🔬Modal Research Exchange

{
  "base_model": "Qwen/Qwen3.5-4B",
  "baseline": "Same loop, seeds, prompts, student settings (lr 1e-4, rank 32, 150 SFT steps of 32 generated tasks) and eval, but the teacher LoRA is never updated, so the frozen teacher generates all tasks.",
  "batch_size": 32,
  "budget": {
    "gpu_hours": 2.0,
    "max_wall_minutes": 120
  },
  "command": "python ued_teacher.py  # custom loop: sc=tinker.ServiceClient(base_url=os.environ['TINKER_BASE_URL']); teacher=sc.create_lora_training_client(base_model='Qwen/Qwen3.5-4B', rank=32); student=sc.create_lora_training_client(base_model='Qwen/Qwen3.5-0.8B', rank=32); for step in range(150): tasks=teacher.save_weights_and_get_sampling_client('t').sample(...32 prompts...); nll0=probe_nll(student); student.forward_backward(sft_datums(tasks),'cross_entropy'); student.optim_step(tinker.AdamParams(learning_rate=1e-4)); r=nll0-probe_nll(student); adv=r-ema; teacher.forward_backward_custom(pg_datums(tasks,adv), is_loss); teacher.optim_step(tinker.AdamParams(learning_rate=2e-5))  # baseline arm: TEACHER_FROZEN=1 skips teacher.optim_step; final eval via student.save_weights_and_get_sampling_client on gsm8k test",
  "dataset": {
    "hf_path": "openai/gsm8k",
    "n_examples": 7473,
    "name": "GSM8K (probe set, seed examples, eval)",
    "split": "train (128 probe + seed prompts) / test (eval)"
  },
  "eval": {
    "benchmark": "GSM8K test (student, greedy, zero-shot)",
    "metric": "exact-match accuracy on final number",
    "n_examples": 1319
  },
  "hypothesis": "A student trained on tasks from a teacher RL-trained on student probe-loss reduction beats a student trained on tasks from the same frozen teacher by at least 3 points of GSM8K test exact-match accuracy.",
  "lora_rank": 32,
  "max_seq_len": 1024,
  "method": "Two LoRA training clients are used. The teacher is Qwen/Qwen3.5-4B, rank 32, trained with RL. The student is Qwen/Qwen3.5-0.8B, rank 32, trained with SFT. Each outer step runs as follows. (1) The teacher samples 32 grade-school math problems, each with a worked solution ending in '#### <number>', from a fixed prompt seeded with 2 random GSM8K train examples. (2) The student takes one SFT step on those 32 problem/solution pairs. (3) Student mean NLL is recomputed on a fixed probe set of 128 GSM8K train examples via logprob calls, and the batch reward is (NLL before the step) minus (NLL after the step). Malformed generations get reward -1. (4) Each task gets advantage = batch reward minus an EMA baseline of past batch rewards, and the teacher takes one importance-sampling policy-gradient step through forward_backward_custom. Everything above is custom client-side loop code; only standard SDK calls are used. The baseline arm runs the identical loop with the teacher's optim_step skipped, so the teacher stays frozen. After 150 outer steps, each final student is evaluated on GSM8K test.",
  "optimizer": {
    "lr": 2e-05,
    "name": "adam"
  },
  "steps": 150,
  "success_criterion": "Validated if the RL-teacher student scores at least 3.0 points higher GSM8K test accuracy than the frozen-teacher student (standard error is about 1.3 points at n=1319) and the teacher's mean batch reward rises over training. Failed if the gap is under 1.5 points or negative.",
  "title": "RL teacher vs frozen teacher generating GSM8K-style SFT data for a 0.8B student",
  "training_type": "rl"
}

rl_student_gsm8k_acc_step140

0.5959

frozen_student_gsm8k_acc_step140

0.6459

gap_points_step140

-5

frozen_student_gsm8k_acc_step150

0.6361

rl_student_gsm8k_acc_step100

0.5876

frozen_student_gsm8k_acc_step100

0.6399

rl_student_gsm8k_acc_step50_run2

0.6285

rl_student_gsm8k_acc_step50_run1

0.652

frozen_student_gsm8k_acc_step50

0.6914

init_student_gsm8k_acc

0.3851

eval_n

1,319

rl_steps_completed

141

frozen_steps_completed

150

rl_mean_batch_reward_steps0_24

0.01064

rl_mean_batch_reward_steps125_140

-0.00016

frozen_mean_batch_reward_steps0_24

0.01036

frozen_mean_batch_reward_steps125_149

-0.0001

rl_teacher_wellformed_frac_steps0_24

0.636

rl_teacher_wellformed_frac_steps125_140

0.998

frozen_teacher_wellformed_frac_overall

0.498

rl_mean_task_reward_steps0_24

-0.358

rl_mean_task_reward_steps125_140

-0.0021