🔬Modal Research Exchange

{
  "activation_gate": {
    "applicable": true,
    "checks": [
      {
        "measures": "model name reported by the created training/sampling client for both arms",
        "metric": "executed_base_model",
        "one_of": [
          "Qwen/Qwen3.5-4B"
        ]
      },
      {
        "max": 0.51,
        "measures": "fraction of the 2000 eval items whose gold verdict is CORRECT",
        "metric": "eval_correct_label_fraction",
        "min": 0.49
      },
      {
        "max": 0,
        "measures": "count of eval PMIDs present in the training items",
        "metric": "train_eval_pmid_overlap",
        "min": 0
      },
      {
        "measures": "relative drop in mean verdict-token train NLL from step 0 to step 20 of the treatment pilot",
        "metric": "verdict_token_loss_drop_pilot",
        "min": 0.1
      },
      {
        "max": 0.95,
        "measures": "fraction of the first 200 eval items the base model labels CORRECT",
        "metric": "base_predicted_correct_fraction",
        "min": 0.05
      }
    ],
    "reason": "Several things must hold before the comparison means anything. The executed model and arms must match the spec. The eval must be balanced and disjoint from training. SFT must actually move the verdict-token loss. The base judge must not be degenerate: if it always answers one class, its 50% is trivial and has to be flagged."
  },
  "arms": [
    {
      "name": "base_untuned",
      "seed": 0,
      "trained": false
    },
    {
      "lr": 0.0001,
      "name": "lora_sft_grader",
      "seed": 0,
      "steps": 150,
      "trained": true
    }
  ],
  "base_model": "Qwen/Qwen3.5-4B",
  "baseline": "Untuned Qwen/Qwen3.5-4B with the same prompt, renderer, eval items and logprob scoring.",
  "batch_size": 32,
  "budget": {
    "gpu_hours": 1.5,
    "max_wall_minutes": 120
  },
  "command": "python grader_sft.py  # tinker SDK; nothing custom except dataset construction and logprob eval\nimport os, random, tinker; from datasets import load_dataset\nfrom tinker_cookbook import renderers; from tinker_cookbook.tokenizer_utils import get_tokenizer\nM='Qwen/Qwen3.5-4B'; svc=tinker.ServiceClient(base_url=os.environ['TINKER_BASE_URL'])\ntok=get_tokenizer(M); r=renderers.get_renderer('qwen3', tok)\ndef items(ds, n, rng):\n  out=[]\n  for ex in list(ds)[:n]:\n    g=ex['final_decision']; w=rng.choice([o for o in ['yes','no','maybe'] if o!=g])\n    for c,lab in [(g,'CORRECT'),(w,'INCORRECT')]:\n      p=f\"Context: {' '.join(ex['context']['contexts'])}\\nQuestion: {ex['question']}\\nCandidate answer: {c}\\nIs the candidate answer correct? Reply CORRECT or INCORRECT.\"\n      out.append((ex['pubid'],p,lab))\n  return out\nrng=random.Random(0)\ntrain=items(load_dataset('qiaojin/PubMedQA','pqa_artificial',split='train').shuffle(seed=0),2400,rng)\nev=items(load_dataset('qiaojin/PubMedQA','pqa_labeled',split='train'),1000,rng)\nassert not {a for a,_,_ in train}&{a for a,_,_ in ev}\nrandom.Random(0).shuffle(train)\ndef datum(p,lab):  # weights only on verdict tokens\n  return renderers.build_supervised_example_datum(r,[{'role':'user','content':p},{'role':'assistant','content':lab}],max_length=1024)  # last-assistant-only weights\ntc=svc.create_lora_training_client(base_model=M, rank=32)\nfor s in range(150):\n  b=[datum(p,l) for _,p,l in train[s*32:(s+1)*32]]\n  tc.forward_backward(b,'cross_entropy').result(); tc.optim_step(tinker.AdamParams(learning_rate=1e-4)).result()\nsc_t=tc.save_weights_and_get_sampling_client('pubmedqa-grader'); sc_b=svc.create_sampling_client(base_model=M)\ndef verdict_lp(sc,p,lab):  # summed logprob of the verdict tokens\n  pre=r.build_generation_prompt([{'role':'user','content':p}]); comp=tok.encode(lab,add_special_tokens=False)\n  mi=tinker.ModelInput.from_ints(pre.to_ints()+comp); lp=sc.compute_logprobs(mi).result(); return sum(lp[-len(comp):])\nfor name,sc in [('baseline',sc_b),('treatment',sc_t)]:\n  acc=[int((verdict_lp(sc,p,'CORRECT')>verdict_lp(sc,p,'INCORRECT'))==(l=='CORRECT')) for _,p,l in ev]\n  print(name, sum(acc)/len(acc))  # save per-item 0/1 for the paired Hoeffding rule",
  "dataset": {
    "hf_path": "qiaojin/PubMedQA",
    "n_examples": 4800,
    "name": "PubMedQA (pqa_artificial train, pqa_labeled eval)",
    "split": "pqa_artificial:train (2400 questions -> 4800 grading items); eval pqa_labeled:train (1000 questions -> 2000 items)"
  },
  "decision_rule": {
    "alpha": 0.05,
    "criteria": [
      {
        "bounds": [
          0,
          1
        ],
        "id": "primary",
        "margin": 0.03,
        "metric": "accuracy",
        "weights": {
          "baseline": -1,
          "treatment": 1
        }
      }
    ],
    "estimand": "Per-item grading accuracy on balanced PubMedQA pqa_labeled grading items: LoRA-SFT Qwen3.5-4B minus untuned Qwen3.5-4B, higher is better; inference conditional on the seed-0 checkpoint",
    "method": "paired_hoeffding",
    "seeds": [
      0
    ],
    "unit": "fixed_checkpoint_items",
    "version": 1
  },
  "eval": {
    "aggregation": null,
    "benchmark": "PubMedQA pqa_labeled balanced grading set (1 correct + 1 incorrect candidate per question)",
    "metric": "grading accuracy (argmax logprob of CORRECT vs INCORRECT vs gold verdict)",
    "n_examples": 2000
  },
  "experiment_features": {
    "adaptive_selection": false,
    "checkpoint_restore": false,
    "custom_loss": false,
    "custom_optimizer": false,
    "generated_training_data": false,
    "heldout_loss": false,
    "learned_generator": false,
    "paired_comparison": true,
    "paired_training_arms": false,
    "warmup_adaptation": false
  },
  "feasibility": {
    "config": {
      "arms": [
        "base_untuned",
        "lora_sft_grader"
      ],
      "base_model": "Qwen/Qwen3.5-4B",
      "batch_size": 32,
      "max_seq_len": 1024,
      "n_eval": 200,
      "steps": 20
    },
    "effective_config": {
      "arms": [
        "base_untuned",
        "lora_sft_grader"
      ],
      "base_model": "Qwen/Qwen3.5-4B",
      "batch_size": 32,
      "max_seq_len": 1024,
      "n_eval": 2000,
      "steps": 150
    },
    "generation": null,
    "measurements": null,
    "primary_endpoint": "pubmedqa_labeled_grading_accuracy",
    "recovery": null,
    "source": null
  },
  "feasibility_check": {
    "generation_applicability": {
      "check": "generation_audit",
      "enforce": false,
      "features": {
        "generated_training_data": false
      },
      "reasons": [
        "spec.generation_audit opts out and no feature it covers is true"
      ],
      "status": "not_applicable"
    },
    "generation_counts": null,
    "gpu_hours": null,
    "known_gpu_hours": 0,
    "phases": {},
    "problems": [
      "Re-projection required: lora_rank changed or was not recorded",
      "Re-projection required: dataset changed or was not recorded",
      "Re-projection required: steps changed or was not recorded",
      "Re-projection required: n_examples changed or was not recorded",
      "Re-projection required: n_eval changed or was not recorded",
      "Pilot evidence source not recorded",
      "Representative setup timing not recorded",
      "Representative training timing not recorded",
      "Representative evaluation timing not recorded",
      "Positive recovery reserve not recorded"
    ],
    "status": "unknown",
    "wall_seconds": null
  },
  "feasibility_required": true,
  "generation_audit": {
    "answer_comparison": "categorical_exact",
    "max_eval_overlaps": 0,
    "max_label_errors": 0,
    "min_sample_size": 100,
    "required": false
  },
  "hypothesis": "LoRA SFT on 4,800 PubMedQA-artificial grading items raises Qwen3.5-4B grading accuracy on balanced expert-labeled PubMedQA grading items by more than 3 points over the untuned base model under the same prompt and scoring, with the lower confidence bound above that margin.",
  "lora_rank": 32,
  "max_seq_len": 1024,
  "method": "Grading items are built deterministically from PubMedQA, with no model-generated data. Each item is (question, abstract contexts, candidate decision in {yes, no, maybe}) with target CORRECT if the candidate equals final_decision, else INCORRECT. Each question gets one correct candidate and one wrong candidate, drawn with seed 0 from the remaining options, so the classes are 50/50 by construction. Training data is 2,400 questions from pqa_artificial (whose labels are auto-derived yes/no), giving 4,800 items. The eval set is all 1,000 expert-labeled pqa_labeled questions, giving 2,000 items. pqa_artificial and pqa_labeled are disjoint by PMID, and this is checked. The treatment is LoRA rank 32 on Qwen/Qwen3.5-4B, trained with a tinker SDK forward_backward + optim_step loop using the stock cross_entropy loss. Loss weight sits only on the verdict tokens. AdamW at lr 1e-4, batch 32, 150 steps (about 1 epoch). The baseline is the untuned Qwen3.5-4B with the identical renderer and prompt. Both are scored by logprob: the predicted verdict is whichever of the 'CORRECT' and 'INCORRECT' completions has the higher summed logprob, computed via forward/compute_logprobs, so there is no sampling noise and no parse failures. The majority-class rate (50%) is reported, along with accuracy on the subsets where the gold label is 'maybe'. Nothing is custom server-side and there is no Spindle fork. correctness_preflight: {\"required\": false, \"reason\": \"stock tinker cross_entropy loss and AdamW; no custom loss or optimizer implementation\"}. Heidi's DPO, batch-size and filtering claims are out of scope for this first node.",
  "optimizer": {
    "lr": 0.0001,
    "name": "adam"
  },
  "primary_endpoint": "pubmedqa_labeled_grading_accuracy",
  "steps": 150,
  "success_criterion": "Treatment minus baseline grading accuracy on the 2000 paired eval items exceeds 0.03, with the paired Hoeffding 95% lower bound above 0.03 (half-width about 0.06 at n=2000, so an observed gain of roughly 0.09 or more is needed). Otherwise the result is not validated, or failed if the point estimate is <= 0.",
  "title": "LoRA-SFT PubMedQA grader vs untuned Qwen3.5-4B judge",
  "training_type": "sft"
}

correctness_validation

{'status': 'not_applicable', 'required': False, 'checks': [], 'reason': 'experiment_features: custom_loss, custom_optimizer is false', 'evidence_source': 'executor report; remote execution is not attested', 'applicability': 'not_applicable', 'reported_verdict': None}

decision_validation

{'status': 'uncertain', 'issues': ['Decision evidence was not recorded.'], 'criteria': [], 'outcome_reason': 'insufficient_evidence', 'unit': 'fixed_checkpoint_items', 'n_seeds': 1, 'alpha': 0.05, 'method': 'paired_hoeffding', 'applicability': {'check': 'decision_rule', 'status': 'required', 'enforce': True, 'reasons': ['experiment_features: paired_comparison is true'], 'features': {'paired_comparison': True, 'adaptive_selection': False}}}

generation_audit_validation

{'status': 'not_applicable', 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'applicability': {'check': 'generation_audit', 'status': 'not_applicable', 'enforce': False, 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'features': {'generated_training_data': False}}}

launch

{'experiment_id': 'exp_8088f9d3ee', 'context_status': 'snapshotted', 'superseded': False, 'mode': 'ordinary', 'purpose': 'Execute the selected saved experiment version.'}

restoration_validation

{'status': 'not_applicable', 'required': False, 'checks': [], 'reason': 'experiment_features: paired_training_arms, checkpoint_restore is false', 'applicability': 'not_applicable'}