{
  "mode": "fresh online CPU sampling and parameter updates on a tiny synthetic-vocabulary LM",
  "method": "fixed-length token-level clipped group-relative objective with sampled KL penalty",
  "prompt": [
    1,
    3
  ],
  "seed": 47,
  "group_size": 16,
  "completion_length": 2,
  "groups": 8,
  "inner_updates": 2,
  "clip": 0.2,
  "kl_beta": 0.04,
  "reward": "1 if the first generated token equals 6; otherwise 0",
  "desired_token_probability_before": 0.11148078965736573,
  "desired_token_probability_after": 0.1291018612493509,
  "sampled_completions": 128,
  "trace": [
    {
      "step": 0,
      "completions": [
        [
          7,
          0
        ],
        [
          1,
          4
        ],
        [
          4,
          7
        ],
        [
          8,
          3
        ],
        [
          2,
          3
        ],
        [
          2,
          7
        ],
        [
          2,
          7
        ],
        [
          0,
          8
        ],
        [
          1,
          7
        ],
        [
          6,
          5
        ],
        [
          8,
          2
        ],
        [
          5,
          4
        ],
        [
          1,
          5
        ],
        [
          7,
          1
        ],
        [
          1,
          5
        ],
        [
          8,
          4
        ]
      ],
      "rewards": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "advantages": [
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        3.8729831862074233,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949
      ],
      "losses": [
        -5.551115123125783e-17,
        -0.003930382077496852
      ],
      "zero_variance": false
    },
    {
      "step": 1,
      "completions": [
        [
          5,
          1
        ],
        [
          5,
          3
        ],
        [
          5,
          2
        ],
        [
          1,
          4
        ],
        [
          6,
          6
        ],
        [
          2,
          8
        ],
        [
          4,
          4
        ],
        [
          6,
          2
        ],
        [
          6,
          6
        ],
        [
          0,
          8
        ],
        [
          0,
          4
        ],
        [
          6,
          5
        ],
        [
          0,
          8
        ],
        [
          6,
          3
        ],
        [
          1,
          1
        ],
        [
          1,
          6
        ]
      ],
      "rewards": [
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        1.0,
        1.0,
        0.0,
        0.0,
        1.0,
        0.0,
        1.0,
        0.0,
        0.0
      ],
      "advantages": [
        -0.6741998479177879,
        -0.6741998479177879,
        -0.6741998479177879,
        -0.6741998479177879,
        1.4832396654191333,
        -0.6741998479177879,
        -0.6741998479177879,
        1.4832396654191333,
        1.4832396654191333,
        -0.6741998479177879,
        -0.6741998479177879,
        1.4832396654191333,
        -0.6741998479177879,
        1.4832396654191333,
        -0.6741998479177879,
        -0.6741998479177879
      ],
      "losses": [
        4.298648638087488e-06,
        -0.013618458296495072
      ],
      "zero_variance": false
    },
    {
      "step": 2,
      "completions": [
        [
          6,
          0
        ],
        [
          3,
          7
        ],
        [
          0,
          4
        ],
        [
          3,
          7
        ],
        [
          8,
          0
        ],
        [
          5,
          4
        ],
        [
          7,
          3
        ],
        [
          1,
          5
        ],
        [
          5,
          2
        ],
        [
          3,
          0
        ],
        [
          7,
          1
        ],
        [
          6,
          8
        ],
        [
          4,
          1
        ],
        [
          4,
          2
        ],
        [
          7,
          7
        ],
        [
          1,
          4
        ]
      ],
      "rewards": [
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "advantages": [
        2.645751231064593,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561,
        2.645751231064593,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561,
        -0.3779644615806561
      ],
      "losses": [
        1.8937726466543292e-05,
        -0.0052481505746524645
      ],
      "zero_variance": false
    },
    {
      "step": 3,
      "completions": [
        [
          8,
          2
        ],
        [
          2,
          7
        ],
        [
          0,
          4
        ],
        [
          4,
          0
        ],
        [
          2,
          1
        ],
        [
          7,
          7
        ],
        [
          8,
          1
        ],
        [
          3,
          4
        ],
        [
          2,
          6
        ],
        [
          3,
          1
        ],
        [
          1,
          1
        ],
        [
          1,
          5
        ],
        [
          1,
          3
        ],
        [
          5,
          1
        ],
        [
          8,
          4
        ],
        [
          2,
          2
        ]
      ],
      "rewards": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "advantages": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "losses": [
        1.7504131772867747e-05,
        1.7477067288182437e-05
      ],
      "zero_variance": true
    },
    {
      "step": 4,
      "completions": [
        [
          1,
          5
        ],
        [
          1,
          3
        ],
        [
          5,
          0
        ],
        [
          0,
          2
        ],
        [
          2,
          6
        ],
        [
          2,
          2
        ],
        [
          4,
          1
        ],
        [
          5,
          0
        ],
        [
          1,
          8
        ],
        [
          1,
          3
        ],
        [
          5,
          4
        ],
        [
          8,
          1
        ],
        [
          5,
          6
        ],
        [
          4,
          7
        ],
        [
          6,
          0
        ],
        [
          8,
          0
        ]
      ],
      "rewards": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0
      ],
      "advantages": [
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        -0.2581988790804949,
        3.8729831862074233,
        -0.2581988790804949
      ],
      "losses": [
        3.07142974115826e-05,
        -0.004061774007876218
      ],
      "zero_variance": false
    },
    {
      "step": 5,
      "completions": [
        [
          5,
          6
        ],
        [
          2,
          2
        ],
        [
          2,
          1
        ],
        [
          7,
          1
        ],
        [
          7,
          4
        ],
        [
          5,
          1
        ],
        [
          8,
          3
        ],
        [
          1,
          1
        ],
        [
          4,
          6
        ],
        [
          7,
          0
        ],
        [
          5,
          3
        ],
        [
          5,
          8
        ],
        [
          0,
          2
        ],
        [
          3,
          3
        ],
        [
          1,
          6
        ],
        [
          7,
          0
        ]
      ],
      "rewards": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "advantages": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "losses": [
        2.3184429881991095e-05,
        2.314783456622477e-05
      ],
      "zero_variance": true
    },
    {
      "step": 6,
      "completions": [
        [
          4,
          4
        ],
        [
          4,
          8
        ],
        [
          2,
          8
        ],
        [
          2,
          4
        ],
        [
          1,
          8
        ],
        [
          5,
          3
        ],
        [
          4,
          6
        ],
        [
          7,
          5
        ],
        [
          5,
          7
        ],
        [
          4,
          5
        ],
        [
          5,
          2
        ],
        [
          0,
          6
        ],
        [
          5,
          7
        ],
        [
          7,
          8
        ],
        [
          7,
          3
        ],
        [
          2,
          0
        ]
      ],
      "rewards": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "advantages": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "losses": [
        1.3303339240217648e-05,
        1.3288557138457224e-05
      ],
      "zero_variance": true
    },
    {
      "step": 7,
      "completions": [
        [
          7,
          3
        ],
        [
          8,
          2
        ],
        [
          8,
          1
        ],
        [
          3,
          3
        ],
        [
          5,
          5
        ],
        [
          8,
          2
        ],
        [
          5,
          7
        ],
        [
          5,
          2
        ],
        [
          3,
          1
        ],
        [
          2,
          2
        ],
        [
          4,
          3
        ],
        [
          2,
          1
        ],
        [
          8,
          2
        ],
        [
          0,
          8
        ],
        [
          5,
          1
        ],
        [
          4,
          4
        ]
      ],
      "rewards": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "advantages": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "losses": [
        7.528826018083634e-06,
        7.523662128849917e-06
      ],
      "zero_variance": true
    }
  ],
  "scope": "No semantic quality, human preference, reasoning or held-out capability claim.",
  "refusal": "finite reward group with at least two members required",
  "command": [
    "python",
    "run.py",
    "grpo",
    "--record"
  ],
  "exit_status": 0,
  "environment": {
    "python": "3.12.8",
    "torch": "2.14.1",
    "platform": "macOS-27.0.1-arm64-arm-64bit"
  },
  "sources": {
    "updates.py": "471ac4ec96c3b092c0c7845a2f812d50dfaac4619fcc2a4bac19c8bd41992f0e",
    "decoder.py": "5706ca21af551b17ee54ca00f5a6011981b327e03b1e880b6712c06dd7d1c39a",
    "vendor/model.py": "0d3d469afd5f35ef19b04e0b0587143c39f8fbca37d6154e6435c4ed4e8b80fc",
    "run.py": "bb3708aaf674c9cd55bd1a21276fb4d4a40e4cd4d15b239f1f994803142d50ce",
    "test_updates.py": "bf71137ae859b1a405941645cdb0db4378fcf55478c548364570778d52186552"
  }
}