🔬Modal Research Exchange

{
  "base_model": "Qwen/Qwen3.5-0.8B",
  "baseline": "Same model, LoRA rank, learning rate, seed, rollout settings and step count, trained with the stock tinker_cookbook math_rl GRPO-style loss.",
  "batch_size": 256,
  "budget": {
    "gpu_hours": 2.0,
    "max_wall_minutes": 120
  },
  "command": "Baseline: python -m tinker_cookbook.recipes.math_rl.train model_name=Qwen/Qwen3.5-0.8B base_url=$TINKER_BASE_URL env=gsm8k group_size=8 groups_per_batch=32 learning_rate=4e-5 max_tokens=512 max_steps=100 log_path=/tmp/rx-dppo-baseline. Treatment (tinker SDK script, custom part is dppo_loss): import os, tinker; sc = tinker.ServiceClient(base_url=os.environ['TINKER_BASE_URL']); tc = sc.create_lora_training_client(base_model='Qwen/Qwen3.5-0.8B', rank=32); for step in range(100): sampler = tc.save_weights_and_get_sampling_client(name=f's{step}'); data, old_logprobs, advantages = rollout_gsm8k(sampler, groups=32, group_size=8, max_tokens=512)  # same rollout and advantage code as the cookbook loop; tc.forward_backward_custom(data, lambda d, logprobs: dppo_loss(logprobs, old_logprobs, advantages)).result()  # dppo_loss implements the arXiv 2602.04879 objective; tc.optim_step(tinker.AdamParams(learning_rate=4e-5)).result(); then tc.save_weights_for_sampler(name='dppo-final') and evaluate both LoRAs on GSM8K test.",
  "dataset": {
    "hf_path": "openai/gsm8k",
    "n_examples": 7473,
    "name": "GSM8K",
    "split": "train"
  },
  "eval": {
    "benchmark": "GSM8K test",
    "metric": "exact-match accuracy, greedy decoding",
    "n_examples": 1319
  },
  "hypothesis": "With the same rollouts, rewards and group-normalized advantages, the DPPO loss raises Qwen3.5-0.8B GSM8K test exact-match accuracy by at least 2 points over the cookbook GRPO-style loss after 100 steps.",
  "lora_rank": 32,
  "max_seq_len": 1024,
  "method": "Train a rank-32 LoRA on Qwen/Qwen3.5-0.8B with RL on the GSM8K train split, using a binary correct-answer reward. Both arms share the same sampling setup: group size 8, 32 groups per batch, max 512 tokens, temperature 1.0, and group-mean advantages. The baseline uses the stock cookbook loss. The treatment changes only the policy loss. It implements the DPPO objective exactly as written in arXiv 2602.04879, as a custom per-token loss through forward_backward_custom, and uses the paper's default hyperparameters. The custom part is the loss function only. Rollouts and advantages reuse the same code as the baseline. Both final LoRAs are evaluated with greedy decoding on the full GSM8K test set. Training reward curves and policy entropy are also logged as a stability check.",
  "optimizer": {
    "lr": 4e-05,
    "name": "adam"
  },
  "steps": 100,
  "success_criterion": "Validated if DPPO accuracy is at least 2.0 points above the baseline on the 1,319 GSM8K test examples, which is beyond the roughly 1.3-point standard error. Failed if DPPO is below the baseline by more than 2.0 points. Anything in between counts as a tie: no accuracy gain, so first-class support would rest on stability or customer demand alone.",
  "title": "DPPO loss vs GRPO loss for LoRA RL on GSM8K",
  "training_type": "rl"
}

base_test_acc

0.7104

grpo_test_acc

0.8544

dppo_test_acc

0.8537

dppo_minus_grpo_pts

-0.07582

n_eval

1,319

grpo_correct

1,127

dppo_correct

1,126

steps_grpo

100

steps_dppo

100

grpo_train_correct_last10

0.8758

dppo_train_correct_last10

0.8637

grpo_entropy_last10

0.1611

dppo_entropy_last10

0.1113

dppo_masked_frac_max

5e-05

grpo_train_minutes

37.9

dppo_train_minutes

41