🔬Modal Research Exchange

{
  "activation_gate": {
    "applicable": true,
    "checks": [
      {
        "max": 0.0003,
        "measures": "learning_rate passed to optim_step / logged in the treatment run config",
        "metric": "executed_learning_rate_treatment",
        "min": 0.0003
      },
      {
        "max": 0.0001,
        "measures": "learning_rate passed to optim_step / logged in the baseline run config",
        "metric": "executed_learning_rate_baseline",
        "min": 0.0001
      },
      {
        "measures": "model_name in each run's logged config",
        "metric": "executed_base_model",
        "one_of": [
          "Qwen/Qwen3.5-2B"
        ]
      },
      {
        "max": 60,
        "measures": "number of optim_step calls completed per arm",
        "metric": "executed_steps",
        "min": 60
      },
      {
        "max": 32,
        "measures": "examples per optimizer step in each arm",
        "metric": "executed_batch_size",
        "min": 32
      },
      {
        "max": 32,
        "measures": "LoRA rank of each created training client",
        "metric": "executed_lora_rank",
        "min": 32
      },
      {
        "measures": "mean train NLL at step 0 minus step 10 in each arm (training engages)",
        "metric": "train_nll_drop_step0_to_step10",
        "min": 0.0
      }
    ],
    "reason": "The only treatment is the learning rate. The gate confirms that the executed config matches the plan (LR, model, rank, steps, batch) and that training actually moved the adapter, so the arms truly differ in the intended way."
  },
  "arms": [
    {
      "lr": 0.0001,
      "name": "baseline_lr1e-4_seed0",
      "seed": 0
    },
    {
      "lr": 0.0003,
      "name": "treatment_lr3e-4_seed0",
      "seed": 0
    },
    {
      "lr": 0.0001,
      "name": "baseline_lr1e-4_seed1",
      "seed": 1
    },
    {
      "lr": 0.0003,
      "name": "treatment_lr3e-4_seed1",
      "seed": 1
    },
    {
      "lr": 0.0001,
      "name": "baseline_lr1e-4_seed2",
      "seed": 2
    },
    {
      "lr": 0.0003,
      "name": "treatment_lr3e-4_seed2",
      "seed": 2
    }
  ],
  "base_model": "Qwen/Qwen3.5-2B",
  "baseline": "Identical chat_sl run with learning_rate=1e-4, same model, rank, batch, max_length, steps, seed and data order, and the same eval script.",
  "batch_size": 32,
  "budget": {
    "gpu_hours": 1.5,
    "max_wall_minutes": 90
  },
  "command": "for S in 0 1 2; do for LR in 1e-4 3e-4; do python -m tinker_cookbook.recipes.chat_sl.train model_name=Qwen/Qwen3.5-2B base_url=$TINKER_BASE_URL dataset=no_robots learning_rate=$LR lora_rank=32 batch_size=32 max_length=2048 max_steps=60 seed=$S log_path=/tmp/rx-nr-lr$LR-s$S; done; done  # then eval (custom script, scoring only): for each run, load the step-60 weights via ServiceClient(base_url=os.environ['TINKER_BASE_URL']).create_lora_training_client(base_model='Qwen/Qwen3.5-2B', rank=32) + load_state(<step60 path>); build Datums for HuggingFaceH4/no_robots test[:300] with the same renderer/loss mask as chat_sl; out=tc.forward(datums,'cross_entropy').result(); per example nll_i=-sum(lp*w), n_i=sum(w) from out.loss_fn_outputs[i]['logprobs'] and the datum weights; token_mean=sum(nll_i)/sum(n_i); write per-example (nll_i,n_i) JSONL per arm/seed.",
  "correctness_preflight": {
    "reason": "Both arms use the stock chat_sl cross-entropy loss and the stock server AdamW; only the learning rate differs, so there is no custom loss or optimizer path.",
    "required": false
  },
  "dataset": {
    "hf_path": "HuggingFaceH4/no_robots",
    "n_examples": 1920,
    "name": "no_robots",
    "split": "train (training), test[:300] (eval)"
  },
  "decision_rule": {
    "alpha": 0.05,
    "assumptions": {
      "identically_distributed_seed_effects": true,
      "independent_seed_effects": true,
      "normal_seed_effects": true
    },
    "criteria": [
      {
        "id": "primary",
        "margin": 0.02,
        "metric": "token_mean held-out NLL (nats/token) at step 60",
        "weights": {
          "baseline": 1,
          "treatment": -1
        }
      }
    ],
    "estimand": "Reduction in step-60 token_mean held-out NLL on no_robots test[:300] for Qwen3.5-2B rank-32 LoRA SFT from using LR 3e-4 instead of 1e-4; positive favors 3e-4",
    "method": "paired_t",
    "seeds": [
      0,
      1,
      2
    ],
    "unit": "training_seed",
    "version": 1
  },
  "eval": {
    "aggregation": "token_mean",
    "benchmark": "no_robots test split, first 300 examples",
    "metric": "token_mean held-out NLL (nats/token) at step 60",
    "n_examples": 300
  },
  "experiment_features": {
    "adaptive_selection": false,
    "checkpoint_restore": false,
    "custom_loss": false,
    "custom_optimizer": false,
    "generated_training_data": false,
    "heldout_loss": true,
    "learned_generator": false,
    "paired_comparison": true,
    "paired_training_arms": true,
    "warmup_adaptation": false
  },
  "feasibility_check": {
    "gpu_hours": null,
    "known_gpu_hours": 0,
    "phases": {},
    "problems": [
      "Measured feasibility not recorded"
    ],
    "status": "unknown",
    "wall_seconds": null
  },
  "feasibility_required": true,
  "generation_audit": {
    "answer_comparison": "categorical_exact",
    "max_eval_overlaps": 0,
    "max_label_errors": 0,
    "min_sample_size": 100,
    "required": false
  },
  "hypothesis": "For rank-32 LoRA SFT of Qwen/Qwen3.5-2B on no_robots, LR 3e-4 gives at least 0.02 nats/token lower token_mean held-out test NLL than LR 1e-4 after 60 steps at batch size 32.",
  "lora_rank": 32,
  "max_seq_len": 2048,
  "method": "Run the stock tinker_cookbook.recipes.chat_sl.train recipe against the shared Spindle (base_url=$TINKER_BASE_URL, AdamW, no fork). There are two arms: baseline lr=1e-4 and treatment lr=3e-4. Both use rank 32, batch 32, max_length 2048 and 60 steps. Within each seed both arms see the same data order. Each arm saves sampler/state weights at step 60. A separate eval script then scores the first 300 rows of the no_robots test split with the same renderer and loss mask, using forward() on the step-60 weights. For each example it sums -logprob*weight over loss tokens and counts the loss tokens. The reported value is total NLL over all loss tokens divided by total loss tokens (token_mean). Batch loss:mean values are never averaged. Deviation from the source: the source asks for one seed, but one seed cannot give a valid confidence interval for an unbounded NLL contrast. This spec therefore schedules 3 training seeds (0, 1, 2). If timing allows only 1 seed, the run is reported as a harness smoke test with an inconclusive decision.",
  "optimizer": {
    "lr": 0.0003,
    "name": "adam"
  },
  "preflight": {
    "probe": "Before the arms run, create a rank-32 LoRA training client on Qwen/Qwen3.5-2B for each arm config. Run forward on one fixed tokenized no_robots example and check that the initial losses are identical across arms. Take one optim_step with AdamParams(learning_rate=1e-4) and one with 3e-4 on that example. Save weights, reload them, and re-run forward on the fixed batch. Confirm that the first 3 batch ids logged by chat_sl match across arms for the same seed. Fail fast if the max_length or seed flag is rejected.",
    "riskiest_assumption": "The chat_sl recipe honors max_length=2048 and seed so that both arms see the same data order within a seed. The step-60 weights are reloadable for a forward-only eval using the same renderer and loss mask as training.",
    "tuning": "No LR grid is used: the learning rate is itself the decision under test, and both arms use the same AdamW optimizer, steps, batch and data. Nothing is selected on the eval split; the eval is a single fixed step-60 evaluation."
  },
  "primary_endpoint": "token_mean held-out NLL (nats/token) on no_robots test[:300] at step 60",
  "restoration_preflight": {
    "applicable": true,
    "fixed_batch_required": true,
    "groups": [
      {
        "arms": [
          {
            "name": "baseline_lr1e-4_seed0",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0001
            }
          },
          {
            "name": "treatment_lr3e-4_seed0",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0003
            }
          }
        ]
      },
      {
        "arms": [
          {
            "name": "baseline_lr1e-4_seed1",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0001
            }
          },
          {
            "name": "treatment_lr3e-4_seed1",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0003
            }
          }
        ]
      },
      {
        "arms": [
          {
            "name": "baseline_lr1e-4_seed2",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0001
            }
          },
          {
            "name": "treatment_lr3e-4_seed2",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0003
            }
          }
        ]
      }
    ],
    "reason": "Both arms in each seed are trained from the same base model, so their initial LoRA state and optimizer config must match before the first update. The arms differ only in LR, which is passed at optim_step."
  },
  "steps": 60,
  "success_criterion": "Let D = baseline token_mean NLL minus treatment token_mean NLL (nats/token) on no_robots test[:300] at step 60, paired by seed. Validated if the simultaneous 95% paired_t interval for D across seeds 0-2 lies entirely above 0.02. Failed if the interval lies entirely below -0.02 (1e-4 better by at least 0.02). Otherwise inconclusive. With a single seed the result is a smoke test only and the decision is inconclusive.",
  "title": "LoRA SFT LR 3e-4 vs 1e-4 on no_robots, Qwen3.5-2B",
  "training_type": "sft"
}

[
  {
    "arm": "baseline_lr1e-4_seed0",
    "step": 60,
    "uri": "tinker://716daf64-43a3-49cf-9306-cebe87f287b6:train:0/weights/final"
  },
  {
    "arm": "treatment_lr3e-4_seed0",
    "step": 60,
    "uri": "tinker://7c44035a-0da1-426b-b1a3-fed1c79adca2:train:0/weights/final"
  },
  {
    "arm": "baseline_lr1e-4_seed1",
    "step": 60,
    "uri": "tinker://6ad9f90f-fa9d-487a-b444-ddb5df06c956:train:0/weights/final"
  },
  {
    "arm": "treatment_lr3e-4_seed1",
    "step": 60,
    "uri": "tinker://4a7402fb-facb-4a17-84e5-c6a8343fbcd1:train:0/weights/final"
  },
  {
    "arm": "baseline_lr1e-4_seed2",
    "step": 60,
    "uri": "tinker://d821fabd-9797-472c-9f5f-77255afd1c89:train:0/weights/final"
  },
  {
    "arm": "treatment_lr3e-4_seed2",
    "step": 60,
    "uri": "tinker://5951599a-b260-4b0a-a5b1-3b23a996a5f8:train:0/weights/final"
  }
]

D_mean_baseline_minus_treatment

0.004607

D_seed0

0.004549

D_seed1

0.003932

D_seed2

0.00534

D_sd

0.0007057

D_ci95_low

0.002854

D_ci95_high

0.00636

margin

0.02

n_eval_examples_per_arm

300

n_eval_loss_tokens_per_arm

59,503

base_init_token_mean_nll

2.439

total_spindle_wall_minutes

32.6

• checkpoint · baseline_lr1e-4_seed0 · step 60 · tinker://716daf64-43a3-49cf-9306-cebe87f287b6:train:0/weights/final
• checkpoint · treatment_lr3e-4_seed0 · step 60 · tinker://7c44035a-0da1-426b-b1a3-fed1c79adca2:train:0/weights/final
• checkpoint · baseline_lr1e-4_seed1 · step 60 · tinker://6ad9f90f-fa9d-487a-b444-ddb5df06c956:train:0/weights/final
• checkpoint · treatment_lr3e-4_seed1 · step 60 · tinker://4a7402fb-facb-4a17-84e5-c6a8343fbcd1:train:0/weights/final
• checkpoint · baseline_lr1e-4_seed2 · step 60 · tinker://d821fabd-9797-472c-9f5f-77255afd1c89:train:0/weights/final
• checkpoint · treatment_lr3e-4_seed2 · step 60 · tinker://5951599a-b260-4b0a-a5b1-3b23a996a5f8:train:0/weights/final
• code_revision · code-revision.txt

1. Nov 3, 23:20 UTC · succeeded · done: baseline_lr1e-4_seed0, treatment_lr3e-4_seed0, baseline_lr1e-4_seed1, treatment_lr3e-4_seed1, baseline_lr1e-4_seed2, treatment_lr3e-4_seed2 · Stock tinker-cookbook chat_sl rank-32 LoRA SFT of Qwen/Qwen3.5-2B on no_robots (batch 32, 60 steps, AdamW, stock linear LR decay) ran on shared Spindle for 3 seeds x 2 LRs, scored on 300 held-out no_…
2. Nov 3, 23:12 UTC · running · done: baseline_lr1e-4_seed0, treatment_lr3e-4_seed0 · Stock chat_sl LoRA SFT (rank 32, batch 32, 60 steps, linear LR decay) of Qwen/Qwen3.5-2B on no_robots on shared Spindle; seed 0 finished with held-out token_mean NLL 2.09594 (lr 1e-4) vs 2.09139 (lr 3e-4).
3. Nov 3, 22:53 UTC · running · Preflight running on shared Spindle (Qwen/Qwen3.5-2B served). Spindle rejects per-model LoRA seeds, so each seed uses a saved random-init checkpoint shared by both arms plus a seed-dependent train shuffle.

correctness_preflight

{'required': False, 'paths': []}

correctness_validation

{'status': 'not_applicable', 'required': False, 'checks': [], 'reason': 'spec.correctness_preflight opts out and no feature it covers is true', 'evidence_source': 'executor report; remote execution is not attested', 'applicability': 'not_applicable', 'reported_verdict': 'inconclusive'}

decision

{'version': 1, 'observations_file': {'name': 'scored-observations.jsonl', 'sha256': 'c2f38ca195dd0f65cf275663989a4d756456ed537bb9a51b1737ef6ff2718ca6'}}

decision_validation

{'status': 'unverified', 'issues': ['Observation file name/hash is not retained for this run.'], 'criteria': [], 'outcome_reason': 'insufficient_evidence', 'applicability': {'check': 'decision_rule', 'status': 'required', 'enforce': True, 'reasons': ['experiment_features: paired_comparison is true'], 'features': {'paired_comparison': True, 'adaptive_selection': False}}}

evaluation

{'version': 1, 'evaluators': {'fd0cc0dfe17a2cf0b7b7e1d3440d68fe976e570883f8232ea32d9be8bc0794ba': {'code_sha256': '8c8453b6bb053c058c6c6a38e98ff2bb2ef3a9c46bf1c090825acbc8508cf083', 'prompt_template': "tinker_cookbook renderer 'qwen3_5', conversation_to_datum(max_length=2048, ALL_ASSISTANT_MESSAGES)", 'decoding': {'temperature': 0, 'max_tokens': 0, 'stop': []}, 'parser': "loss_eval.LossEval('token_mean')", 'items': {'manifest_ref': 'evaluator_provenance.json'}}}}

evaluator_validation

{'status': 'invalid', 'issues': ['Expected evaluators, scores and comparisons.'], 'comparisons': []}

feasibility

{'status': 'unknown', 'problems': ['Re-projection required: lora_rank changed or was not recorded', 'Re-projection required: dataset changed or was not recorded', 'Re-projection required: steps changed or was not recorded', 'Re-projection required: arms changed or was not recorded', 'Re-projection required: n_examples changed or was not recorded', 'Re-projection required: n_eval changed or was not recorded', 'Positive recovery reserve not recorded'], 'wall_seconds': None, 'gpu_hours': None, 'known_gpu_hours': 0, 'phases': {'setup': {'wall_seconds': 252.7, 'planned_units': 1, 'planned_samples': 0.0, 'gpu_hours': None}, 'training': {'wall_seconds': 711.45, 'planned_units': 360, 'planned_samples': 11520.0, 'gpu_hours': None}, 'evaluation': {'wall_seconds': 147.0, 'planned_units': 2100, 'planned_samples': 2100.0, 'gpu_hours': None}}, 'generation_counts': None, 'generation_applicability': {'check': 'generation_audit', 'status': 'not_applicable', 'enforce': False, 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'features': {'generated_training_data': False}}, 'record': {'config': {'base_model': 'Qwen/Qwen3.5-2B', 'batch_size': 32, 'max_seq_len': 2048, 'pilot': '2 batch-32 steps + 1 forward of 32'}, 'effective_config': {'base_model': 'Qwen/Qwen3.5-2B', 'arms': 6, 'steps': 60, 'batch_size': 32, 'max_seq_len': 2048, 'n_eval': 300}, 'primary_endpoint': 'token_mean held-out NLL (nats/token) on no_robots test[:300] at step 60', 'source': 'preflight_out.json, runs/*/arm_timing.json', 'measurements': {'setup': {'seconds': 252.7, 'units': 1, 'planned_units': 1, 'samples': 0, 'gpu_count': None, 'allocation_source': None}, 'training': {'seconds': 711.45, 'units': 360, 'planned_units': 360, 'samples': 11520, 'gpu_count': None, 'allocation_source': None}, 'evaluation': {'seconds': 21, 'units': 300, 'planned_units': 2100, 'samples': 300, 'gpu_count': None, 'allocation_source': None}}, 'generation': None, 'recovery': {'seconds': 0, 'gpu_count': None, 'allocation_source': None}}, 'recorded_at': 1793748023.8097074}

feasibility_required

True

generation_audit_validation

{'status': 'not_applicable', 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'applicability': {'check': 'generation_audit', 'status': 'not_applicable', 'enforce': False, 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'features': {'generated_training_data': False}}}

initialization_provenance

{'version': 1, 'note': 'Adapter sha256 unavailable: save_weights_external returns 404 on Spindle (and stalls the client request sequence). Proof is a shared full initialization checkpoint per seed loaded by both arms via create_training_client_from_state; seed-0 pair has bit-identical full initial logprobs on a fixed batch; all pairs have bit-identical step-0 train NLL on the same 32-example batch.', 'groups': {'seed0': {'checkpoint_uri': 'tinker://8ff281af876a6bb849f1395cb6de405e/weights/init-seed0', 'initial_output': {'available': True, 'batch_sha256': '6ff46d26c02471d39a04fa86173632086f809558cd1beb0e50b9306fb7ba6fd4', 'output_sha256': '8a5c5514e09b80a4153feb85b9a72c16e73b0356bef6e0d56bec0d8c620403bf', 'match': True, 'evidence_ref': 'preflight_out.json'}}, 'seed1': {'checkpoint_uri': 'tinker://85df6b5ab377e88eeed4d70230fd33d8/weights/init-seed1', 'initial_output': {'available': False, 'reason': 'only step-0 train NLL captured (identical: 2.6985340118408203)'}}, 'seed2': {'checkpoint_uri': 'tinker://f2be07d0279f2ac2090fcf159e5e3e20/weights/init-seed2', 'initial_output': {'available': False, 'reason': 'only step-0 train NLL captured (identical: 2.4172232151031494)'}}}, 'optimizer_checks': "get_info().optimizer_config after create/restore = {'type': 'adamw'} (server_response, preflight); betas (0.9, 0.95), eps 1e-8 declared via AdamParams, not exposed by server", 'restore_count': 0}

last_reply

cf275663989a4d756456ed537bb9a51b1737ef6ff2718ca6"}}, "feasibility": {"config": {"base_model": "Qwen/Qwen3.5-2B", "batch_size": 32, "max_seq_len": 2048, "pilot": "2 batch-32 steps + 1 forward of 32"}, "effective_config": {"base_model": "Qwen/Qwen3.5-2B", "arms": 6, "steps": 60, "batch_size": 32, "max_seq_len": 2048, "n_eval": 300}, "primary_endpoint": "token_mean held-out NLL (nats/token) on no_robots test[:300] at step 60", "source": "preflight_out.json, runs/*/arm_timing.json", "measurements": {"setup": {"seconds": 252.7, "units": 1, "planned_units": 1, "samples": 0, "gpu_count": null, "allocation_source": null}, "training": {"seconds": 711.45, "units": 360, "planned_units": 360, "samples": 11520, "gpu_count": null, "allocation_source": null}, "evaluation": {"seconds": 21, "units": 300, "planned_units": 2100, "samples": 300, "gpu_count": null, "allocation_source": null}}, "generation": null, "recovery": {"seconds": 0, "gpu_count": null, "allocation_source": null}}, "evidence": {"version": 1, "files": [{"name": "code-revision.txt", "kind": "code_revision", "content": "tinker-cookbook 6daabe0838736557f0e802020b710073eb340287 (no local changes); tinker SDK 0.37.0; shared Spindle, no fork"}], "checkpoints": [ {"arm": "baseline_lr1e-4_seed0", "step": 60, "uri": "tinker://716daf64-43a3-49cf-9306-cebe87f287b6:train:0/weights/final"}, {"arm": "treatment_lr3e-4_seed0", "step": 60, "uri": "tinker://7c44035a-0da1-426b-b1a3-fed1c79adca2:train:0/weights/final"}, {"arm": "baseline_lr1e-4_seed1", "step": 60, "uri": "tinker://6ad9f90f-fa9d-487a-b444-ddb5df06c956:train:0/weights/final"}, {"arm": "treatment_lr3e-4_seed1", "step": 60, "uri": "tinker://4a7402fb-facb-4a17-84e5-c6a8343fbcd1:train:0/weights/final"}, {"arm": "baseline_lr1e-4_seed2", "step": 60, "uri": "tinker://d821fabd-9797-472c-9f5f-77255afd1c89:train:0/weights/final"}, {"arm": "treatment_lr3e-4_seed2", "step": 60, "uri": "tinker://5951599a-b260-4b0a-a5b1-3b23a996a5f8:train:0/weights/final"}]}} ```

last_sent_at

1793747674.9767761

launch

{'experiment_id': 'exp_9574bbb76e', 'context_status': 'snapshotted', 'superseded': False, 'mode': 'ordinary', 'purpose': 'Execute the selected saved experiment version.'}

progress_updated_at

1793747949.3316891

reported_outcome_reason

tie

reported_verdict

inconclusive

restoration_validation

{'status': 'unverified', 'required': True, 'checks': [{'check': 'baseline_lr1e-4_seed0', 'status': 'unverified', 'reason': 'baseline_lr1e-4_seed0: missing or duplicate arm evidence.'}, {'check': 'treatment_lr3e-4_seed0', 'status': 'unverified', 'reason': 'treatment_lr3e-4_seed0: missing or duplicate arm evidence.'}, {'check': 'baseline_lr1e-4_seed1', 'status': 'unverified', 'reason': 'baseline_lr1e-4_seed1: missing or duplicate arm evidence.'}, {'check': 'treatment_lr3e-4_seed1', 'status': 'unverified', 'reason': 'treatment_lr3e-4_seed1: missing or duplicate arm evidence.'}, {'check': 'baseline_lr1e-4_seed2', 'status': 'unverified', 'reason': 'baseline_lr1e-4_seed2: missing or duplicate arm evidence.'}, {'check': 'treatment_lr3e-4_seed2', 'status': 'unverified', 'reason': 'treatment_lr3e-4_seed2: missing or duplicate arm evidence.'}], 'reason': 'baseline_lr1e-4_seed0: missing or duplicate arm evidence.', 'applicability': 'required'}

started_at

1793745384.3941097