🔬Modal Research Exchange

{
  "base_model": "Qwen/Qwen3.5-2B",
  "baseline": "Unedited Qwen/Qwen3.5-2B, scored on the same 500 GSM8K test problems with the identical prompt, renderer, greedy decoding and 512-token limit.",
  "batch_size": 200,
  "budget": {
    "gpu_hours": 2.0,
    "max_wall_minutes": 150
  },
  "command": "python edit_search.py  # tinker SDK: sc = tinker.ServiceClient(base_url=os.environ['TINKER_BASE_URL']); agent = sc.create_sampling_client(base_model='Qwen/Qwen3.8-27B'); for it in range(50): edit = propose(agent, history); handle = fork_weight_write('Qwen/Qwen3.5-2B', edit)  # CUSTOM: endpoint added in the Spindle fork; target = sc.create_sampling_client(model_path=handle); score = gsm8k_eval(target, dev200); keep edit if score improves; final: gsm8k_eval on test500 for edited model and for sc.create_sampling_client(base_model='Qwen/Qwen3.5-2B')",
  "dataset": {
    "hf_path": "openai/gsm8k",
    "n_examples": 700,
    "name": "gsm8k",
    "split": "train (200 dev for search), test (500 held-out eval)"
  },
  "eval": {
    "benchmark": "GSM8K test",
    "metric": "exact-match accuracy on final numeric answer",
    "n_examples": 500
  },
  "hypothesis": "Qwen3.5-2B after agent-chosen direct weight edits touching at most 0.016% of parameters beats the unedited Qwen3.5-2B by at least 5 points exact-match on 500 held-out GSM8K test problems.",
  "lora_rank": 0,
  "max_seq_len": 1024,
  "method": "Needs a Spindle fork that adds a weight-write API: an endpoint that takes (tensor name, index set, new values) and writes them into a copy of the served model's weights, then exposes that copy to the sampling client. Qwen3.5-0.8B is not served, so Qwen/Qwen3.5-2B, the closest served model, is used. The edit budget is 0.016% of parameters, about 320K values. The editor agent is Qwen/Qwen3.8-27B through the sampling client. It sees the layer and tensor names, the current dev score and failure examples, and proposes one sparse edit per iteration, such as scaling selected rows or columns, adding small offsets, or changing norm gains. Each edit is applied through the fork and scored on 200 GSM8K train problems used as a dev set. An edit is kept only if the dev score improves, and edits accumulate within the budget. The loop runs for 50 iterations with no gradients. The final edited model and the unedited baseline are then scored on 500 GSM8K test problems, never seen during the search. Both use the same chat template, the same 4-shot-free zero-shot prompt with a 'final answer after ####' instruction, greedy decoding and max 512 tokens. No LoRA is trained.",
  "optimizer": {
    "lr": 0.0,
    "name": "none (agent-driven direct weight edits via fork weight-write API)"
  },
  "steps": 50,
  "success_criterion": "Validated if the edited model's GSM8K test exact-match exceeds the baseline by at least 5.0 points, roughly 2x the ~2.2-point standard error at n=500, and all edits stay within 320K parameters. Failed if the gain is under 2 points. Anything in between is inconclusive.",
  "title": "Agent sparse weight edits vs unedited Qwen3.5-2B on GSM8K",
  "training_type": "eval_only"
}

base_dev_acc

0.77

best_dev_acc

0.81

iterations_run

50

edits_accepted

3

values_touched

49,152

base_test_acc

0.744

edited_test_acc

0.734

base_test_correct

372

edited_test_correct

367

test_n

500

delta_points

-1

wall_minutes_search_and_eval

24.3