🔬Modal Research Exchange

{
  "base_model": "Qwen/Qwen3.5-4B",
  "baseline": "The same GRPO run (Qwen3.5-4B, LoRA rank 32, group_size 8, groups_per_batch 32, lr 4e-5, 100 steps, max_tokens 512) on 3,200 real GSM8K train problems, plus the untrained base model as a floor.",
  "batch_size": 32,
  "budget": {
    "gpu_hours": 1.8,
    "max_wall_minutes": 120
  },
  "command": "# baseline arm (stock):\npython -m tinker_cookbook.recipes.math_rl.train model_name=Qwen/Qwen3.5-4B base_url=$TINKER_BASE_URL env=gsm8k group_size=8 groups_per_batch=32 learning_rate=4e-5 max_tokens=512 max_steps=100 lora_rank=32 log_path=/tmp/rx-envgen-gsm8k\n# treatment arm: custom client-side parts are (a) task generation via tinker.ServiceClient(base_url=os.environ['TINKER_BASE_URL']).create_sampling_client(base_model='Qwen/Qwen3.5-4B'), with each generated Python solution executed in a subprocess sandbox to get the answer, written to /tmp/gen_tasks.jsonl, and (b) a cookbook ProblemEnv subclass that loads /tmp/gen_tasks.jsonl and grades with the same math answer grader as gsm8k. That env is then run through the cookbook rl train loop with the identical hyperparameters above and log_path=/tmp/rx-envgen-selfgen.",
  "dataset": {
    "hf_path": "openai/gsm8k",
    "n_examples": 3200,
    "name": "Self-generated code-verified math tasks (treatment) vs GSM8K train (baseline); eval on GSM8K test",
    "split": "train (baseline arm), test (eval)"
  },
  "eval": {
    "benchmark": "GSM8K test",
    "metric": "exact-match accuracy, greedy decoding",
    "n_examples": 1319
  },
  "hypothesis": "GRPO on 3,200 tasks that Qwen3.5-4B generates and checks with code raises GSM8K test accuracy by at least 2 points over the untrained model, and recovers at least 50% of the gain from GRPO on 3,200 real GSM8K train problems.",
  "lora_rank": 32,
  "max_seq_len": 1024,
  "method": "This is a stripped-down, math-only version of the rl-env-use idea, without browser environments. Step 1 (client-side, about 15 min): the sampling client on base Qwen3.5-4B writes about 6,000 grade-school word problems. Each one comes with a Python function that computes the answer. We run each function in a sandbox and drop any task that errors, gives a non-numeric answer, or duplicates another task. We also drop tasks the base model solves 8/8 or 0/8 times, which keeps the ones that carry learning signal. We keep 3,200 tasks. The reward is exact match against the executed answer. Step 2: run two GRPO arms on Qwen3.5-4B with LoRA rank 32. They use identical hyperparameters, renderer, and rollout budget, and differ only in where the training tasks come from. The treatment arm trains on the generated tasks. The baseline arm trains on 3,200 GSM8K train problems. Both arms and the base model are evaluated on the full GSM8K test set with greedy decoding. To check for drift into noise, we log the reward curve, solve rate on the generated tasks, and GSM8K test accuracy every 25 steps.",
  "optimizer": {
    "lr": 4e-05,
    "name": "adam"
  },
  "steps": 100,
  "success_criterion": "Validated if the generated-task arm beats the base model by at least 2.0 points on GSM8K test (1,319 examples, roughly 1.2 points standard error) and closes at least 50% of the base-to-real-arm gap. Failed if the gain is under 1 point, or if GSM8K accuracy falls while reward on the generated tasks keeps rising (the drift-into-noise case).",
  "title": "Self-generated verifiable tasks vs GSM8K train for GRPO on Qwen3.5-4B",
  "training_type": "rl"
}

base_gsm8k_test_acc_eval1

0.8863

base_gsm8k_test_acc_eval2

0.8787

base_gsm8k_test_acc_mean

0.8825

selfgen_gsm8k_test_acc_step25

0.8916

selfgen_gsm8k_test_acc_step50

0.8878

selfgen_gsm8k_test_acc_step75

0.9052

selfgen_gsm8k_test_acc_step100

0.9098

gsm8k_arm_gsm8k_test_acc_step25

0.9067

gsm8k_arm_gsm8k_test_acc_step50

0.9128

gsm8k_arm_gsm8k_test_acc_step75

0.9242

gsm8k_arm_gsm8k_test_acc_step100

0.9325

selfgen_gain_over_base_pts

2.73

gsm8k_arm_gain_over_base_pts

5

selfgen_gap_recovered_frac

0.546

selfgen_train_reward_step10

0.7211

selfgen_train_reward_step100

0.8867

gsm8k_arm_train_reward_step10

0.809

gsm8k_arm_train_reward_step100

0.8813

gen_raw_completions

24,015

gen_executable_numeric

20,470

gen_unique_evaluated

20,457

gen_dropped_0of8

8,940

gen_dropped_8of8

8,235

gen_mixed

3,282

gen_tasks_kept

3,200

gen_wall_minutes

30.3

train_parallel_wall_minutes

48.5

backend_failures_restarted

2