{
 "title": "Score centering vs plain GRPO on GSM8K at an aggressive learning rate",
 "hypothesis": "With LoRA GRPO on Qwen3.5-0.8B at a deliberately high learning rate, score centering limits the drop from peak to final GSM8K test accuracy to at most 2 points, while the uncentered baseline drops by at least 5 points.",
 "method": "Both arms use one tinker SDK GRPO script that differs only by a CENTER flag. Each step samples 32 GSM8K train problems with 8 rollouts each from the current LoRA and scores exact-match reward. Advantages are reward minus the group mean, and the importance-sampling policy-gradient loss goes through forward_backward_custom. With CENTER=1, the loss applies the score-centering correction exactly as written in Martin's paper. That correction is the only custom part, and it must be copied from the paper's equations before launch. With CENTER=0 the loss is the stock GRPO surrogate. Every 10 steps the script saves sampler weights and evaluates greedy exact match on 500 fixed GSM8K test problems. Every step it logs mean reward, gradient norm, sampler-vs-learner logprob gap and token entropy as stability signals. Each arm runs 2 seeds at learning rate 1e-4, chosen so the baseline is likely to destabilize.",
 "base_model": "Qwen/Qwen3.5-0.8B",
 "lora_rank": 32,
 "training_type": "rl",
 "dataset": {
  "name": "GSM8K",
  "hf_path": "openai/gsm8k",
  "split": "main/train (train), main/test first 500 (eval)",
  "n_examples": 3200
 },
 "optimizer": {
  "name": "adam",
  "lr": 0.0001
 },
 "steps": 100,
 "batch_size": 256,
 "max_seq_len": 1024,
 "eval": {
  "benchmark": "GSM8K test (first 500)",
  "metric": "greedy exact-match accuracy; peak-minus-final drop over the run",
  "n_examples": 500
 },
 "baseline": "Same script, model, seeds, data order, group size 8, 32 groups per batch, max_tokens 512, LoRA rank 32, AdamW lr 1e-4, and 100 steps, with CENTER=0 (standard group-mean-centered GRPO surrogate, no score centering).",
 "success_criterion": "Validated if, averaged over 2 seeds, the treatment's peak-minus-final accuracy drop is \u22642 points and is \u22653 points smaller than the baseline's, and final accuracy is not more than 1.5 points below baseline. Failed if the drops differ by <3 points, which is within seed noise on 500 examples.",
 "budget": {
  "gpu_hours": 2.0,
  "max_wall_minutes": 120
 },
 "command": "CENTER=0 SEED=0 python rl_score_centering.py && CENTER=1 SEED=0 python rl_score_centering.py (repeat SEED=1)  # rl_score_centering.py: sc=tinker.ServiceClient(base_url=os.environ['TINKER_BASE_URL']); tc=sc.create_lora_training_client(base_model='Qwen/Qwen3.5-0.8B', rank=32); loop 100 steps: samp=tc.save_weights_and_get_sampling_client(name=f'step{i}'); sample 32 GSM8K prompts x8 (max_tokens=512); reward=exact match; adv=r-mean(group); tc.forward_backward_custom(data, loss_fn) where loss_fn is the IS policy-gradient loss plus the score-centering term when CENTER=1 (custom part); tc.optim_step(tinker.AdamParams(learning_rate=1e-4)); every 10 steps eval 500 GSM8K test greedy; log grad norm, logprob gap, entropy to /tmp/rx-sc-$CENTER-$SEED",
 "correctness_preflight": {
  "required": true,
  "paths": [
   {
    "id": "score_centering_loss",
    "kind": "loss",
    "baseline_revision": "tinker-0.37.0 built-in loss_fn=importance_sampling (stock GRPO IS surrogate) on shared Spindle",
    "reference_revision": "github.com/martin-marek/score-centering@fdca22b19793babdf08e33e5ba51537e82cbed2f README score_centering_loss (JAX) + losses.py loss_fn_rl",
    "tolerances": {
     "baseline_loss": {
      "atol": 0.0,
      "rtol": 0.01
     },
     "baseline_gradient": {
      "atol": 1e-06,
      "rtol": 1e-05,
      "min_cosine": 0.9999
     },
     "reference_loss": {
      "atol": 1e-06,
      "rtol": 0.0001
     },
     "reference_gradient": {
      "atol": 1e-07,
      "rtol": 0.0001,
      "min_cosine": 0.9999
     }
    }
   }
  ]
 }
}