🔬Modal Research Exchange

{
  "base_model": "Qwen/Qwen3.5-0.8B",
  "baseline": "Same script, model, seeds, data order, group size 8, 32 groups per batch, max_tokens 512, LoRA rank 32, AdamW lr 1e-4, and 100 steps, with CENTER=0 (standard group-mean-centered GRPO surrogate, no score centering).",
  "batch_size": 256,
  "budget": {
    "gpu_hours": 2.0,
    "max_wall_minutes": 120
  },
  "command": "CENTER=0 SEED=0 python rl_score_centering.py && CENTER=1 SEED=0 python rl_score_centering.py (repeat SEED=1)  # rl_score_centering.py: sc=tinker.ServiceClient(base_url=os.environ['TINKER_BASE_URL']); tc=sc.create_lora_training_client(base_model='Qwen/Qwen3.5-0.8B', rank=32); loop 100 steps: samp=tc.save_weights_and_get_sampling_client(name=f'step{i}'); sample 32 GSM8K prompts x8 (max_tokens=512); reward=exact match; adv=r-mean(group); tc.forward_backward_custom(data, loss_fn) where loss_fn is the IS policy-gradient loss plus the score-centering term when CENTER=1 (custom part); tc.optim_step(tinker.AdamParams(learning_rate=1e-4)); every 10 steps eval 500 GSM8K test greedy; log grad norm, logprob gap, entropy to /tmp/rx-sc-$CENTER-$SEED",
  "dataset": {
    "hf_path": "openai/gsm8k",
    "n_examples": 3200,
    "name": "GSM8K",
    "split": "main/train (train), main/test first 500 (eval)"
  },
  "eval": {
    "benchmark": "GSM8K test (first 500)",
    "metric": "greedy exact-match accuracy; peak-minus-final drop over the run",
    "n_examples": 500
  },
  "hypothesis": "With LoRA GRPO on Qwen3.5-0.8B at a deliberately high learning rate, score centering limits the drop from peak to final GSM8K test accuracy to at most 2 points, while the uncentered baseline drops by at least 5 points.",
  "lora_rank": 32,
  "max_seq_len": 1024,
  "method": "Both arms use one tinker SDK GRPO script that differs only by a CENTER flag. Each step samples 32 GSM8K train problems with 8 rollouts each from the current LoRA and scores exact-match reward. Advantages are reward minus the group mean, and the importance-sampling policy-gradient loss goes through forward_backward_custom. With CENTER=1, the loss applies the score-centering correction exactly as written in Martin's paper. That correction is the only custom part, and it must be copied from the paper's equations before launch. With CENTER=0 the loss is the stock GRPO surrogate. Every 10 steps the script saves sampler weights and evaluates greedy exact match on 500 fixed GSM8K test problems. Every step it logs mean reward, gradient norm, sampler-vs-learner logprob gap and token entropy as stability signals. Each arm runs 2 seeds at learning rate 1e-4, chosen so the baseline is likely to destabilize.",
  "optimizer": {
    "lr": 0.0001,
    "name": "adam"
  },
  "steps": 100,
  "success_criterion": "Validated if, averaged over 2 seeds, the treatment's peak-minus-final accuracy drop is \u22642 points and is \u22653 points smaller than the baseline's, and final accuracy is not more than 1.5 points below baseline. Failed if the drops differ by <3 points, which is within seed noise on 500 examples.",
  "title": "Score centering vs plain GRPO on GSM8K at an aggressive learning rate",
  "training_type": "rl"
}

base_peak_acc

0.861

base_final_acc

0.832

base_drop

0.029

sc_peak_acc

0.858

sc_final_acc

0.847

sc_drop

0.011

drop_diff_base_minus_sc

0.018

final_diff_sc_minus_base

0.015

base_s0_drop

0.052

base_s1_drop

0.006

sc_s0_drop

0.02

sc_s1_drop

0.002

base_s0_final

0.822

base_s1_final

0.842

sc_s0_final

0.84

sc_s1_final

0.854

base_init_acc

0.747

sc_init_acc

0.746

base_grad_norm_mean

0.0324

sc_grad_norm_mean

0.0263

base_grad_norm_max

0.0658

sc_grad_norm_max

0.0431

base_logprob_gap_abs

0.0059

sc_logprob_gap_abs

0.0062

base_entropy_last10

0.0896

sc_entropy_last10

0.1914

base_reward_last10

0.849

sc_reward_last10

0.8686

sc_topk_k

32

wall_minutes

77