🔬Modal Research Exchange

{
  "activation_gate": {
    "applicable": true,
    "checks": [
      {
        "measures": "lora_rank reported in the training client config / logged run config for rank-1 runs",
        "metric": "executed_lora_rank_rank1_arm",
        "one_of": [
          "1"
        ]
      },
      {
        "measures": "lora_rank reported in the training client config / logged run config for rank-32 runs",
        "metric": "executed_lora_rank_rank32_arm",
        "one_of": [
          "32"
        ]
      },
      {
        "measures": "fraction of the first step's 8 groups whose 8 rewards are not all equal",
        "metric": "frac_groups_nonzero_advantage_step0",
        "min": 0.25
      },
      {
        "max": 0.85,
        "measures": "mean correctness reward of base-model rollouts at step 0",
        "metric": "mean_train_reward_step0",
        "min": 0.05
      }
    ],
    "reason": "The treatment is a rank change, so the executed adapter rank must be verified. Policy gradient also only learns when groups have mixed rewards, so the pilot must show a nonzero advantage signal and some reward headroom for both arms."
  },
  "arms": [
    {
      "lora_rank": 1,
      "lr": 0.0001,
      "name": "rank1_lr1e-4",
      "seed": 0
    },
    {
      "lora_rank": 1,
      "lr": 0.0003,
      "name": "rank1_lr3e-4",
      "seed": 0
    },
    {
      "lora_rank": 1,
      "lr": 0.001,
      "name": "rank1_lr1e-3",
      "seed": 0
    },
    {
      "lora_rank": 32,
      "lr": 0.0001,
      "name": "rank32_lr1e-4",
      "seed": 0
    },
    {
      "lora_rank": 32,
      "lr": 0.0003,
      "name": "rank32_lr3e-4",
      "seed": 0
    },
    {
      "lora_rank": 32,
      "lr": 0.001,
      "name": "rank32_lr1e-3",
      "seed": 0
    }
  ],
  "base_model": "Qwen/Qwen2.5-3B",
  "baseline": "Rank-32 LoRA on the same model, recipe, data order, group size, batch, max_tokens, steps and seed, with its own LR grid {1e-4, 3e-4, 1e-3}. The best LR is chosen on test items 0-199.",
  "batch_size": 8,
  "budget": {
    "gpu_hours": 2.0,
    "max_wall_minutes": 120
  },
  "command": "# 1) Pilot, before the sweep: run rank1_lr3e-4 and rank32_lr3e-4 for 3 steps, check activation_gate (executed rank, nonzero-advantage fraction, step-0 reward) and record feasibility timing; do not change settings without a recorded deviation.\n# 2) Train six runs (stock recipe, no custom parts):\nfor R in 1 32; do for LR in 1e-4 3e-4 1e-3; do python -m tinker_cookbook.recipes.math_rl.train model_name=Qwen/Qwen2.5-3B base_url=$TINKER_BASE_URL env=gsm8k lora_rank=$R group_size=8 groups_per_batch=8 learning_rate=$LR max_tokens=512 max_steps=40 seed=0 log_path=/tmp/rx-lwr/r${R}_lr${LR}; done; done\n# 3) Eval (short custom script on the tinker SDK): sampling_client = service.create_sampling_client(model_path=<final sampler path>); greedy (temperature=0) max_tokens=512 with the recipe's GSM8K prompt/renderer; parse the final number and compare by numeric exact match. Score test[0:200] for all 6 runs plus the base model, then pick the best LR per rank (a tie goes to the lower LR). If the pick is on a grid edge, add 3e-5 or 3e-3 for that rank only if the measured feasibility allows; otherwise mark the run inconclusive. Freeze the choice, then score test[200:1319] once for the two selected checkpoints and the base, and save per-item correctness for the paired Hoeffding CI.\n# Activation-failure plan: if the step-0 nonzero-advantage fraction is <0.25 or the reward is outside [0.05,0.85], keep the failed pilot log, record a revised spec (e.g. max_tokens or prompt fix from the cookbook defaults) before any rerun, and never relax the gate.",
  "correctness_preflight": {
    "reason": "Both arms use the unchanged math_rl recipe loss and the stock AdamW optimizer. Only lora_rank and LR differ, and no custom loss or optimizer is implemented.",
    "required": false
  },
  "dataset": {
    "hf_path": "openai/gsm8k",
    "n_examples": 7473,
    "name": "gsm8k",
    "split": "train (training) / test (selection items 0-199, confirmation items 200-1318)"
  },
  "decision_rule": {
    "adaptive_selection": {
      "confirmation_item_ids": "gsm8k/test[200:1319]",
      "kind": "hyperparameter",
      "rationale": "Each rank's LR is chosen from {1e-4, 3e-4, 1e-3} by accuracy on test items 0-199 and frozen before the single confirmation eval on items 200-1318.",
      "search_item_ids": "gsm8k/test[0:200]"
    },
    "alpha": 0.05,
    "criteria": [
      {
        "bounds": [
          0,
          1
        ],
        "id": "primary_noninferiority",
        "margin": -0.09,
        "metric": "accuracy",
        "weights": {
          "baseline": -1,
          "treatment": 1
        }
      }
    ],
    "estimand": "Difference in GSM8K test (items 200-1318) greedy exact-match accuracy between the LR-tuned rank-1 and LR-tuned rank-32 LoRA RL checkpoints on Qwen2.5-3B, at a fixed 40-step endpoint; larger means rank-1 is better",
    "method": "paired_hoeffding",
    "seeds": [
      0
    ],
    "unit": "fixed_checkpoint_items",
    "version": 1
  },
  "eval": {
    "aggregation": null,
    "benchmark": "GSM8K test items 200-1318 (confirmation)",
    "metric": "greedy numeric exact-match accuracy",
    "n_examples": 1119
  },
  "experiment_features": {
    "adaptive_selection": true,
    "checkpoint_restore": false,
    "custom_loss": false,
    "custom_optimizer": false,
    "generated_training_data": false,
    "heldout_loss": false,
    "learned_generator": false,
    "paired_comparison": true,
    "paired_training_arms": true,
    "warmup_adaptation": false
  },
  "feasibility_check": {
    "gpu_hours": null,
    "known_gpu_hours": 0,
    "phases": {},
    "problems": [
      "Measured feasibility not recorded"
    ],
    "status": "unknown",
    "wall_seconds": null
  },
  "feasibility_required": true,
  "generation_audit": {
    "answer_comparison": "numeric_exact",
    "max_eval_overlaps": 0,
    "max_label_errors": 0,
    "min_sample_size": 100,
    "required": false
  },
  "hypothesis": "With each arm at its best learning rate, rank-1 LoRA GRPO on Qwen2.5-3B base reaches GSM8K confirmation accuracy no more than 9 points below rank-32 LoRA, using the lower bound of a simultaneous 95% CI. Rank-1 at least matching rank-32 is the post's claim.",
  "lora_rank": 1,
  "max_seq_len": 1024,
  "method": "Scaled-down version of the post's RL setup, which ran GSM8K/MATH on Llama-3.1-8B base with group-mean-subtracted advantages and an importance-sampled policy gradient. Llama is not served here, so we use Qwen/Qwen2.5-3B, a served base model. Training uses the stock tinker_cookbook.recipes.math_rl.train with env=gsm8k and the recipe's default importance-sampling loss. Settings: constant LR, AdamW, group_size=8, groups_per_batch=8, max_tokens=512, 40 steps. The two arms differ only in lora_rank (1 vs 32), and the cookbook default alpha/init is used for both. Each rank runs an LR grid of {1e-4, 3e-4, 1e-3}, bracketing the post's roughly 10x-FullFT LoRA LR for a model with hidden size 2048. All six runs use seed 0 and the same environment ordering. Each final checkpoint is scored greedily on GSM8K test with numeric exact match. Items 0-199 select each arm's LR, and items 200-1318 are the single confirmation evaluation. The FullFT comparison from the post is out of scope: Spindle is LoRA-only, and no fork is planned. This tests only the post's 'rank-1 suffices for RL' claim, not 'LoRA matches FullFT'.",
  "optimizer": {
    "lr": 0.0003,
    "name": "adam"
  },
  "preflight": {
    "probe": "Create training clients for Qwen/Qwen2.5-3B with lora_rank=1 and lora_rank=32. For each, run forward on a fixed tokenized batch and check the logprobs match each other (B=0 init). Then take one forward_backward plus optim_step(tinker.AdamParams(learning_rate=1e-3, beta1=0.9, beta2=0.95, eps=1e-8)) on one GSM8K example and confirm both succeed and the logged rank is the requested one.",
    "riskiest_assumption": "math_rl on Spindle actually honors lora_rank=1 for Qwen2.5-3B (a valid rank-1 adapter on all layers, not silently clamped), and the base model gets enough mixed-reward GSM8K groups in 512 tokens for policy gradient to learn within 40 steps.",
    "tuning": "Each rank gets its own grid {1e-4, 3e-4, 1e-3} under an identical 40-step, 8x8-rollout budget. Selection is on GSM8K test items 0-199, disjoint from confirmation items 200-1318. If a best LR sits on a grid edge, the grid is extended one point for that arm if feasibility allows; otherwise the comparison is inconclusive, never a win."
  },
  "primary_endpoint": "gsm8k_test_confirmation_exact_match_accuracy",
  "restoration_preflight": {
    "applicable": true,
    "fixed_batch_required": true,
    "groups": [
      {
        "arms": [
          {
            "name": "rank1_lr1e-4",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0001
            }
          },
          {
            "name": "rank1_lr3e-4",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0003
            }
          },
          {
            "name": "rank1_lr1e-3",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.001
            }
          },
          {
            "name": "rank32_lr1e-4",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0001
            }
          },
          {
            "name": "rank32_lr3e-4",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.0003
            }
          },
          {
            "name": "rank32_lr1e-3",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 0.001
            }
          }
        ]
      }
    ],
    "reason": "Trained arms share the Qwen2.5-3B base with freshly initialized LoRA (B=0), so both arms start from the same function. Before the first update, verify this with identical logprobs on a fixed tokenized batch, and verify each client's active AdamW config after creation. Server identity beyond this stays unverified."
  },
  "spec_preflight": {
    "findings": [],
    "status": "passed"
  },
  "steps": 40,
  "success_criterion": "Validated if the simultaneous 95% paired Hoeffding CI lower bound on (best-rank1 minus best-rank32) GSM8K confirmation accuracy over 1119 items exceeds -0.09, and both selected arms beat the untrained base by at least 0.05 point estimate. Failed if the CI upper bound is below -0.09. Otherwise the result is inconclusive, and so is a selected LR on the grid edge that cannot be extended.",
  "title": "Rank-1 vs rank-32 LoRA under GRPO-style RL on GSM8K, best LR vs best LR",
  "training_type": "rl"
}

[
  {
    "arm": "rank1_lr1e-3",
    "step": 40,
    "uri": "tinker://b1503a4ce8731de97e8dc361cde0a152:train:0/sampler_weights/final"
  },
  {
    "arm": "rank1_lr1e-4",
    "step": 40,
    "uri": "tinker://36d478cb73c0d4af71be9e84880ce6fc:train:0/sampler_weights/final"
  },
  {
    "arm": "rank1_lr3e-4",
    "step": 40,
    "uri": "tinker://1293cfb24019e3fc1998a35d984bddf0:train:0/sampler_weights/final"
  },
  {
    "arm": "rank32_lr1e-3",
    "step": 40,
    "uri": "tinker://fa768cbf5c7fd3a1554d3f4319b2478f:train:0/sampler_weights/final"
  },
  {
    "arm": "rank32_lr3e-4",
    "step": 40,
    "uri": "tinker://2aa88565-ba3e-4d6b-abc3-c0f9777792da:train:0/sampler_weights/final"
  },
  {
    "arm": "rank32_lr1e-4",
    "step": 40,
    "uri": "tinker://4d7ec04e-f22a-4ee5-9c17-64cb108c82e2:train:0/sampler_weights/final"
  },
  {
    "arm": "rank1_lr3e-5",
    "step": 40,
    "uri": "tinker://080365acb1f7ca92af4613520ce33213:train:0/sampler_weights/final"
  }
]

{
  "base": {
    "confirmation_accuracy": 0.7345844504021448,
    "selection_accuracy": 0.71
  },
  "baseline": {
    "confirmation_accuracy": 0.8105451295799821
  },
  "rank1_lr1e-3": {
    "final_sampler_checkpoint_step": 40,
    "frac_mixed_step0": 1.0,
    "learning_rate": 0.001,
    "lora_rank": 1,
    "selection_accuracy": 0.0,
    "train_correct_last": 0.0,
    "train_correct_step0": 0.6875,
    "train_steps_completed": 40
  },
  "rank1_lr1e-4": {
    "final_sampler_checkpoint_step": 40,
    "frac_mixed_step0": 1.0,
    "learning_rate": 0.0001,
    "lora_rank": 1,
    "selection_accuracy": 0.82,
    "train_correct_last": 0.875,
    "train_correct_step0": 0.6875,
    "train_steps_completed": 40
  },
  "rank1_lr3e-4": {
    "final_sampler_checkpoint_step": 40,
    "frac_mixed_step0": 1.0,
    "learning_rate": 0.0003,
    "lora_rank": 1,
    "selection_accuracy": 0.82,
    "train_correct_last": 0.8125,
    "train_correct_step0": 0.703125,
    "train_steps_completed": 40
  },
  "rank1_lr3e-5": {
    "confirmation_accuracy": 0.8239499553172476,
    "final_sampler_checkpoint_step": 40,
    "frac_mixed_step0": 1.0,
    "learning_rate": 3e-05,
    "lora_rank": 1,
    "selection_accuracy": 0.82,
    "train_correct_last": 0.671875,
    "train_correct_step0": 0.75,
    "train_steps_completed": 40
  },
  "rank32_lr1e-3": {
    "final_sampler_checkpoint_step": 40,
    "frac_mixed_step0": 1.0,
    "learning_rate": 0.001,
    "lora_rank": 32,
    "selection_accuracy": 0.0,
    "train_correct_last": 0.0,
    "train_correct_step0": 0.640625,
    "train_steps_completed": 40
  },
  "rank32_lr1e-4": {
    "final_sampler_checkpoint_step": 40,
    "frac_mixed_step0": 1.0,
    "learning_rate": 0.0001,
    "lora_rank": 32,
    "selection_accuracy": 0.815,
    "train_correct_last": 0.828125,
    "train_correct_step0": 0.75,
    "train_steps_completed": 40
  },
  "rank32_lr3e-4": {
    "confirmation_accuracy": 0.8105451295799821,
    "final_sampler_checkpoint_step": 40,
    "frac_mixed_step0": 1.0,
    "learning_rate": 0.0003,
    "lora_rank": 32,
    "selection_accuracy": 0.84,
    "train_correct_last": 0.828125,
    "train_correct_step0": 0.65625,
    "train_steps_completed": 40
  },
  "treatment": {
    "confirmation_accuracy": 0.8239499553172476
  }
}

base_selection_accuracy

0.71

base_confirmation_accuracy

0.7346

effect_rank1_minus_rank32

0.0134

ci_lower

-0.06779

ci_upper

0.0946

treatment_confirmation_accuracy

0.8239

baseline_confirmation_accuracy

0.8105

treatment_minus_base

0.08937

baseline_minus_base

0.07596

n_confirmation_items

1,119

selected_lr_rank1

3e-05

selected_lr_rank32

0.0003

• checkpoint · rank1_lr1e-3 · step 40 · tinker://b1503a4ce8731de97e8dc361cde0a152:train:0/sampler_weights/final
• checkpoint · rank1_lr1e-4 · step 40 · tinker://36d478cb73c0d4af71be9e84880ce6fc:train:0/sampler_weights/final
• checkpoint · rank1_lr3e-4 · step 40 · tinker://1293cfb24019e3fc1998a35d984bddf0:train:0/sampler_weights/final
• checkpoint · rank32_lr1e-3 · step 40 · tinker://fa768cbf5c7fd3a1554d3f4319b2478f:train:0/sampler_weights/final
• checkpoint · rank32_lr3e-4 · step 40 · tinker://2aa88565-ba3e-4d6b-abc3-c0f9777792da:train:0/sampler_weights/final
• checkpoint · rank32_lr1e-4 · step 40 · tinker://4d7ec04e-f22a-4ee5-9c17-64cb108c82e2:train:0/sampler_weights/final
• checkpoint · rank1_lr3e-5 · step 40 · tinker://080365acb1f7ca92af4613520ce33213:train:0/sampler_weights/final
• script · assemble.py sha256 5e5f4578397a
• script · auto_sel.sh sha256 524eff32be30
• script · build_report.py sha256 763e7cd89fbe
• code_revision · code-revision.json sha256 a05da5568122
• script · common.py sha256 f6918e60cb26
• metrics · eval-scores-and-selection.json sha256 13eedf41edc6
• script · eval_gsm8k.py sha256 c30bf146eab8
• script · final.py sha256 dc26cb3f4136
• metrics · fixed-batch-and-initial-logprobs.json sha256 8b6a1e1ab2d2
• metrics · init-provenance-bundle.json sha256 c095713b717a
• metrics · logs-excerpts.json sha256 666cedf2c4d5
• predictions · pred_confirmation_base.jsonl sha256 7b1b3d44c47c
• predictions · pred_confirmation_rank1_lr3e-5.jsonl sha256 c94c33f636f3
• predictions · pred_confirmation_rank32_lr3e-4.jsonl sha256 859d77511b5e
• predictions · pred_selection_base.jsonl sha256 f744e7e37f52
• predictions · pred_selection_rank1_lr1e-3.jsonl sha256 74990592ff78
• predictions · pred_selection_rank1_lr1e-4.jsonl sha256 5f2975d090e7
• predictions · pred_selection_rank1_lr3e-4.jsonl sha256 8a7caef40ebd
• predictions · pred_selection_rank1_lr3e-5.jsonl sha256 3b76f1b66c0c
• predictions · pred_selection_rank32_lr1e-3.jsonl sha256 3d276212765f
• predictions · pred_selection_rank32_lr1e-4.jsonl sha256 285b6d96857a
• predictions · pred_selection_rank32_lr3e-4.jsonl sha256 047e15b1cb7e
• metrics · probe-results.json sha256 fe9d306e6d3e
• script · probe.py sha256 10ccfc225fb9
• metrics · scored-observations.jsonl sha256 0429c3d0dea7
• script · select_lr.py sha256 829d732b07dc
• script · train_arm.py sha256 e5b6586647ce
• metrics · training-metrics.json sha256 488d1f2367d8
• effective_config · effective-config.json

1. Nov 4, 18:04 UTC · succeeded · done: rank1_lr1e-4, rank1_lr3e-4, rank1_lr1e-3, rank32_lr1e-4, rank32_lr3e-4, rank32_lr1e-3, rank1_lr3e-5 · Stock tinker-cookbook math_rl GRPO-style (importance-sampling) runs on Qwen/Qwen2.5-3B on shared Spindle: 7 LoRA arms over 40 steps (rank 1 at LR 3e-5/1e-4/3e-4/1e-3, rank 32 at 1e-4/3e-4/1e-3), with…
2. Nov 4, 17:48 UTC · running · done: rank1_lr1e-4, rank1_lr3e-4, rank1_lr1e-3, rank32_lr1e-4, rank32_lr3e-4, rank32_lr1e-3 · All six 40-step arms finished. Selection on test[0:200] picked rank32 lr 3e-4 (0.84) and, for rank1, a 1e-4/3e-4 tie at 0.82 that lands on the grid edge, so a 3e-5 rank1 extension arm is training now…
3. Nov 4, 17:34 UTC · running · done: rank1_lr1e-3 · Six 40-step stock math_rl arms on Qwen/Qwen2.5-3B are training on shared Spindle. rank1_lr1e-3 finished but collapsed (test[0:200] accuracy 0.0). Two rank-32 arms hit 410 model_lost and are resuming …
4. Nov 4, 17:15 UTC · running · Preflight probe and 3-step pilots passed (executed ranks 1/32, bitwise-identical initial logprobs, activation gate in range); six 40-step stock math_rl arms on Qwen/Qwen2.5-3B are now training on sha…
5. Nov 4, 16:53 UTC · running · Launch instructions downloaded and hash-verified; Qwen/Qwen2.5-3B confirmed served on shared Spindle. Running the preflight probe next.

correctness_preflight

{'required': False, 'paths': []}

correctness_validation

{'status': 'not_applicable', 'required': False, 'checks': [], 'reason': 'spec.correctness_preflight opts out and no feature it covers is true', 'evidence_source': 'executor report; remote execution is not attested', 'applicability': 'not_applicable', 'reported_verdict': 'refuted'}

decision

{'version': 1, 'observations_file': {'name': 'scored-observations.jsonl', 'sha256': '0429c3d0dea700eec3f1a27ac532a636fba050a373612ff02ff4c68cea44147a'}, 'selection': {'frozen_at_unix': 1793815150.7161703, 'frozen_before_confirmation': True, 'ranks': {'1': {'candidates': {'1e-3': 0.0, '1e-4': 0.82, '3e-4': 0.82, '3e-5': 0.82}, 'grid_edge': True, 'selected_accuracy': 0.82, 'selected_arm': 'rank1_lr3e-5', 'selected_lr': '3e-5'}, '32': {'candidates': {'1e-3': 0.0, '1e-4': 0.815, '3e-4': 0.84}, 'grid_edge': False, 'selected_accuracy': 0.84, 'selected_arm': 'rank32_lr3e-4', 'selected_lr': '3e-4'}}, 'rule': 'argmax test[0:200] greedy accuracy per rank; ties -> lower LR; grid edge -> extend once with 3e-5/3e-3 if feasible', 'selection_evaluator_id': 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052', 'search_item_ids': 'gsm8k/test/0..199', 'confirmation_item_ids': 'gsm8k/test/200..1318', 'treatment_arm_checkpoint': 'rank1_lr3e-5', 'baseline_arm_checkpoint': 'rank32_lr3e-4'}}

decision_validation

{'status': 'unverified', 'issues': ['Observation file name/hash is not retained for this run.'], 'criteria': [], 'outcome_reason': 'insufficient_evidence', 'applicability': {'check': 'decision_rule', 'status': 'required', 'enforce': True, 'reasons': ['experiment_features: paired_comparison, adaptive_selection is true'], 'features': {'paired_comparison': True, 'adaptive_selection': True}}, 'reported_verdict': 'refuted'}

evaluation

{'version': 1, 'evaluators': {'f1d45345e6712abaa28f75c82cbb2800cdb101b45ec44e5d87e02d7ff47b3023': {'code_sha256': '0b46931e371b503139a3dc29febd95819f24059f6bab50e477354666bfad5036', 'prompt_template': {'renderer': 'role_colon', 'tokenizer': 'Qwen/Qwen2.5-3B', 'convo': 'MathEnv.standard_fewshot_prefix() + [user: question + Gsm8kDataset.question_suffix()]', 'question_suffix': ' Provide a numerical answer without units, written inside \\boxed{}.', 'build': 'renderer.build_generation_prompt(convo)'}, 'decoding': {'temperature': 0.0, 'max_tokens': 512, 'stop': 'renderer.get_stop_sequences() (role_colon)'}, 'parser': {'code': 'eval_gsm8k.parse/correct', 'rule': "last number in last \\boxed{} (else last number in text), strip ',' '$', float; correct iff finite and == float(reference '####' answer)"}, 'items': {'ids': ['gsm8k/test/200', 'gsm8k/test/201', 'gsm8k/test/202', 'gsm8k/test/203', 'gsm8k/test/204', 'gsm8k/test/205', 'gsm8k/test/206', 'gsm8k/test/207', 'gsm8k/test/208', 'gsm8k/test/209', 'gsm8k/test/210', 'gsm8k/test/211', 'gsm8k/test/212', 'gsm8k/test/213', 'gsm8k/test/214', 'gsm8k/test/215', 'gsm8k/test/216', 'gsm8k/test/217', 'gsm8k/test/218', 'gsm8k/test/219', 'gsm8k/test/220', 'gsm8k/test/221', 'gsm8k/test/222', 'gsm8k/test/223', 'gsm8k/test/224', 'gsm8k/test/225', 'gsm8k/test/226', 'gsm8k/test/227', 'gsm8k/test/228', 'gsm8k/test/229', 'gsm8k/test/230', 'gsm8k/test/231', 'gsm8k/test/232', 'gsm8k/test/233', 'gsm8k/test/234', 'gsm8k/test/235', 'gsm8k/test/236', 'gsm8k/test/237', 'gsm8k/test/238', 'gsm8k/test/239', 'gsm8k/test/240', 'gsm8k/test/241', 'gsm8k/test/242', 'gsm8k/test/243', 'gsm8k/test/244', 'gsm8k/test/245', 'gsm8k/test/246', 'gsm8k/test/247', 'gsm8k/test/248', 'gsm8k/test/249', 'gsm8k/test/250', 'gsm8k/test/251', 'gsm8k/test/252', 'gsm8k/test/253', 'gsm8k/test/254', 'gsm8k/test/255', 'gsm8k/test/256', 'gsm8k/test/257', 'gsm8k/test/258', 'gsm8k/test/259', 'gsm8k/test/260', 'gsm8k/test/261', 'gsm8k/test/262', 'gsm8k/test/263', 'gsm8k/test/264', 'gsm8k/test/265', 'gsm8k/test/266', 'gsm8k/test/267', 'gsm8k/test/268', 'gsm8k/test/269', 'gsm8k/test/270', 'gsm8k/test/271', 'gsm8k/test/272', 'gsm8k/test/273', 'gsm8k/test/274', 'gsm8k/test/275', 'gsm8k/test/276', 'gsm8k/test/277', 'gsm8k/test/278', 'gsm8k/test/279', 'gsm8k/test/280', 'gsm8k/test/281', 'gsm8k/test/282', 'gsm8k/test/283', 'gsm8k/test/284', 'gsm8k/test/285', 'gsm8k/test/286', 'gsm8k/test/287', 'gsm8k/test/288', 'gsm8k/test/289', 'gsm8k/test/290', 'gsm8k/test/291', 'gsm8k/test/292', 'gsm8k/test/293', 'gsm8k/test/294', 'gsm8k/test/295', 'gsm8k/test/296', 'gsm8k/test/297', 'gsm8k/test/298', 'gsm8k/test/299', 'gsm8k/test/300', 'gsm8k/test/301', 'gsm8k/test/302', 'gsm8k/test/303', 'gsm8k/test/304', 'gsm8k/test/305', 'gsm8k/test/306', 'gsm8k/test/307', 'gsm8k/test/308', 'gsm8k/test/309', 'gsm8k/test/310', 'gsm8k/test/311', 'gsm8k/test/312', 'gsm8k/test/313', 'gsm8k/test/314', 'gsm8k/test/315', 'gsm8k/test/316', 'gsm8k/test/317', 'gsm8k/test/318', 'gsm8k/test/319', 'gsm8k/test/320', 'gsm8k/test/321', 'gsm8k/test/322', 'gsm8k/test/323', 'gsm8k/test/324', 'gsm8k/test/325', 'gsm8k/test/326', 'gsm8k/test/327', 'gsm8k/test/328', 'gsm8k/test/329', 'gsm8k/test/330', 'gsm8k/test/331', 'gsm8k/test/332', 'gsm8k/test/333', 'gsm8k/test/334', 'gsm8k/test/335', 'gsm8k/test/336', 'gsm8k/test/337', 'gsm8k/test/338', 'gsm8k/test/339', 'gsm8k/test/340', 'gsm8k/test/341', 'gsm8k/test/342', 'gsm8k/test/343', 'gsm8k/test/344', 'gsm8k/test/345', 'gsm8k/test/346', 'gsm8k/test/347', 'gsm8k/test/348', 'gsm8k/test/349', 'gsm8k/test/350', 'gsm8k/test/351', 'gsm8k/test/352', 'gsm8k/test/353', 'gsm8k/test/354', 'gsm8k/test/355', 'gsm8k/test/356', 'gsm8k/test/357', 'gsm8k/test/358', 'gsm8k/test/359', 'gsm8k/test/360', 'gsm8k/test/361', 'gsm8k/test/362', 'gsm8k/test/363', 'gsm8k/test/364', 'gsm8k/test/365', 'gsm8k/test/366', 'gsm8k/test/367', 'gsm8k/test/368', 'gsm8k/test/369', 'gsm8k/test/370', 'gsm8k/test/371', 'gsm8k/test/372', 'gsm8k/test/373', 'gsm8k/test/374', 'gsm8k/test/375', 'gsm8k/test/376', 'gsm8k/test/377', 'gsm8k/test/378', 'gsm8k/test/379', 'gsm8k/test/380', 'gsm8k/test/381', 'gsm8k/test/382', 'gsm8k/test/383', 'gsm8k/test/384', 'gsm8k/test/385', 'gsm8k/test/386', 'gsm8k/test/387', 'gsm8k/test/388', 'gsm8k/test/389', 'gsm8k/test/390', 'gsm8k/test/391', 'gsm8k/test/392', 'gsm8k/test/393', 'gsm8k/test/394', 'gsm8k/test/395', 'gsm8k/test/396', 'gsm8k/test/397', 'gsm8k/test/398', 'gsm8k/test/399', 'gsm8k/test/400', 'gsm8k/test/401', 'gsm8k/test/402', 'gsm8k/test/403', 'gsm8k/test/404', 'gsm8k/test/405', 'gsm8k/test/406', 'gsm8k/test/407', 'gsm8k/test/408', 'gsm8k/test/409', 'gsm8k/test/410', 'gsm8k/test/411', 'gsm8k/test/412', 'gsm8k/test/413', 'gsm8k/test/414', 'gsm8k/test/415', 'gsm8k/test/416', 'gsm8k/test/417', 'gsm8k/test/418', 'gsm8k/test/419', 'gsm8k/test/420', 'gsm8k/test/421', 'gsm8k/test/422', 'gsm8k/test/423', 'gsm8k/test/424', 'gsm8k/test/425', 'gsm8k/test/426', 'gsm8k/test/427', 'gsm8k/test/428', 'gsm8k/test/429', 'gsm8k/test/430', 'gsm8k/test/431', 'gsm8k/test/432', 'gsm8k/test/433', 'gsm8k/test/434', 'gsm8k/test/435', 'gsm8k/test/436', 'gsm8k/test/437', 'gsm8k/test/438', 'gsm8k/test/439', 'gsm8k/test/440', 'gsm8k/test/441', 'gsm8k/test/442', 'gsm8k/test/443', 'gsm8k/test/444', 'gsm8k/test/445', 'gsm8k/test/446', 'gsm8k/test/447', 'gsm8k/test/448', 'gsm8k/test/449', 'gsm8k/test/450', 'gsm8k/test/451', 'gsm8k/test/452', 'gsm8k/test/453', 'gsm8k/test/454', 'gsm8k/test/455', 'gsm8k/test/456', 'gsm8k/test/457', 'gsm8k/test/458', 'gsm8k/test/459', 'gsm8k/test/460', 'gsm8k/test/461', 'gsm8k/test/462', 'gsm8k/test/463', 'gsm8k/test/464', 'gsm8k/test/465', 'gsm8k/test/466', 'gsm8k/test/467', 'gsm8k/test/468', 'gsm8k/test/469', 'gsm8k/test/470', 'gsm8k/test/471', 'gsm8k/test/472', 'gsm8k/test/473', 'gsm8k/test/474', 'gsm8k/test/475', 'gsm8k/test/476', 'gsm8k/test/477', 'gsm8k/test/478', 'gsm8k/test/479', 'gsm8k/test/480', 'gsm8k/test/481', 'gsm8k/test/482', 'gsm8k/test/483', 'gsm8k/test/484', 'gsm8k/test/485', 'gsm8k/test/486', 'gsm8k/test/487', 'gsm8k/test/488', 'gsm8k/test/489', 'gsm8k/test/490', 'gsm8k/test/491', 'gsm8k/test/492', 'gsm8k/test/493', 'gsm8k/test/494', 'gsm8k/test/495', 'gsm8k/test/496', 'gsm8k/test/497', 'gsm8k/test/498', 'gsm8k/test/499', 'gsm8k/test/500', 'gsm8k/test/501', 'gsm8k/test/502', 'gsm8k/test/503', 'gsm8k/test/504', 'gsm8k/test/505', 'gsm8k/test/506', 'gsm8k/test/507', 'gsm8k/test/508', 'gsm8k/test/509', 'gsm8k/test/510', 'gsm8k/test/511', 'gsm8k/test/512', 'gsm8k/test/513', 'gsm8k/test/514', 'gsm8k/test/515', 'gsm8k/test/516', 'gsm8k/test/517', 'gsm8k/test/518', 'gsm8k/test/519', 'gsm8k/test/520', 'gsm8k/test/521', 'gsm8k/test/522', 'gsm8k/test/523', 'gsm8k/test/524', 'gsm8k/test/525', 'gsm8k/test/526', 'gsm8k/test/527', 'gsm8k/test/528', 'gsm8k/test/529', 'gsm8k/test/530', 'gsm8k/test/531', 'gsm8k/test/532', 'gsm8k/test/533', 'gsm8k/test/534', 'gsm8k/test/535', 'gsm8k/test/536', 'gsm8k/test/537', 'gsm8k/test/538', 'gsm8k/test/539', 'gsm8k/test/540', 'gsm8k/test/541', 'gsm8k/test/542', 'gsm8k/test/543', 'gsm8k/test/544', 'gsm8k/test/545', 'gsm8k/test/546', 'gsm8k/test/547', 'gsm8k/test/548', 'gsm8k/test/549', 'gsm8k/test/550', 'gsm8k/test/551', 'gsm8k/test/552', 'gsm8k/test/553', 'gsm8k/test/554', 'gsm8k/test/555', 'gsm8k/test/556', 'gsm8k/test/557', 'gsm8k/test/558', 'gsm8k/test/559', 'gsm8k/test/560', 'gsm8k/test/561', 'gsm8k/test/562', 'gsm8k/test/563', 'gsm8k/test/564', 'gsm8k/test/565', 'gsm8k/test/566', 'gsm8k/test/567', 'gsm8k/test/568', 'gsm8k/test/569', 'gsm8k/test/570', 'gsm8k/test/571', 'gsm8k/test/572', 'gsm8k/test/573', 'gsm8k/test/574', 'gsm8k/test/575', 'gsm8k/test/576', 'gsm8k/test/577', 'gsm8k/test/578', 'gsm8k/test/579', 'gsm8k/test/580', 'gsm8k/test/581', 'gsm8k/test/582', 'gsm8k/test/583', 'gsm8k/test/584', 'gsm8k/test/585', 'gsm8k/test/586', 'gsm8k/test/587', 'gsm8k/test/588', 'gsm8k/test/589', 'gsm8k/test/590', 'gsm8k/test/591', 'gsm8k/test/592', 'gsm8k/test/593', 'gsm8k/test/594', 'gsm8k/test/595', 'gsm8k/test/596', 'gsm8k/test/597', 'gsm8k/test/598', 'gsm8k/test/599', 'gsm8k/test/600', 'gsm8k/test/601', 'gsm8k/test/602', 'gsm8k/test/603', 'gsm8k/test/604', 'gsm8k/test/605', 'gsm8k/test/606', 'gsm8k/test/607', 'gsm8k/test/608', 'gsm8k/test/609', 'gsm8k/test/610', 'gsm8k/test/611', 'gsm8k/test/612', 'gsm8k/test/613', 'gsm8k/test/614', 'gsm8k/test/615', 'gsm8k/test/616', 'gsm8k/test/617', 'gsm8k/test/618', 'gsm8k/test/619', 'gsm8k/test/620', 'gsm8k/test/621', 'gsm8k/test/622', 'gsm8k/test/623', 'gsm8k/test/624', 'gsm8k/test/625', 'gsm8k/test/626', 'gsm8k/test/627', 'gsm8k/test/628', 'gsm8k/test/629', 'gsm8k/test/630', 'gsm8k/test/631', 'gsm8k/test/632', 'gsm8k/test/633', 'gsm8k/test/634', 'gsm8k/test/635', 'gsm8k/test/636', 'gsm8k/test/637', 'gsm8k/test/638', 'gsm8k/test/639', 'gsm8k/test/640', 'gsm8k/test/641', 'gsm8k/test/642', 'gsm8k/test/643', 'gsm8k/test/644', 'gsm8k/test/645', 'gsm8k/test/646', 'gsm8k/test/647', 'gsm8k/test/648', 'gsm8k/test/649', 'gsm8k/test/650', 'gsm8k/test/651', 'gsm8k/test/652', 'gsm8k/test/653', 'gsm8k/test/654', 'gsm8k/test/655', 'gsm8k/test/656', 'gsm8k/test/657', 'gsm8k/test/658', 'gsm8k/test/659', 'gsm8k/test/660', 'gsm8k/test/661', 'gsm8k/test/662', 'gsm8k/test/663', 'gsm8k/test/664', 'gsm8k/test/665', 'gsm8k/test/666', 'gsm8k/test/667', 'gsm8k/test/668', 'gsm8k/test/669', 'gsm8k/test/670', 'gsm8k/test/671', 'gsm8k/test/672', 'gsm8k/test/673', 'gsm8k/test/674', 'gsm8k/test/675', 'gsm8k/test/676', 'gsm8k/test/677', 'gsm8k/test/678', 'gsm8k/test/679', 'gsm8k/test/680', 'gsm8k/test/681', 'gsm8k/test/682', 'gsm8k/test/683', 'gsm8k/test/684', 'gsm8k/test/685', 'gsm8k/test/686', 'gsm8k/test/687', 'gsm8k/test/688', 'gsm8k/test/689', 'gsm8k/test/690', 'gsm8k/test/691', 'gsm8k/test/692', 'gsm8k/test/693', 'gsm8k/test/694', 'gsm8k/test/695', 'gsm8k/test/696', 'gsm8k/test/697', 'gsm8k/test/698', 'gsm8k/test/699', 'gsm8k/test/700', 'gsm8k/test/701', 'gsm8k/test/702', 'gsm8k/test/703', 'gsm8k/test/704', 'gsm8k/test/705', 'gsm8k/test/706', 'gsm8k/test/707', 'gsm8k/test/708', 'gsm8k/test/709', 'gsm8k/test/710', 'gsm8k/test/711', 'gsm8k/test/712', 'gsm8k/test/713', 'gsm8k/test/714', 'gsm8k/test/715', 'gsm8k/test/716', 'gsm8k/test/717', 'gsm8k/test/718', 'gsm8k/test/719', 'gsm8k/test/720', 'gsm8k/test/721', 'gsm8k/test/722', 'gsm8k/test/723', 'gsm8k/test/724', 'gsm8k/test/725', 'gsm8k/test/726', 'gsm8k/test/727', 'gsm8k/test/728', 'gsm8k/test/729', 'gsm8k/test/730', 'gsm8k/test/731', 'gsm8k/test/732', 'gsm8k/test/733', 'gsm8k/test/734', 'gsm8k/test/735', 'gsm8k/test/736', 'gsm8k/test/737', 'gsm8k/test/738', 'gsm8k/test/739', 'gsm8k/test/740', 'gsm8k/test/741', 'gsm8k/test/742', 'gsm8k/test/743', 'gsm8k/test/744', 'gsm8k/test/745', 'gsm8k/test/746', 'gsm8k/test/747', 'gsm8k/test/748', 'gsm8k/test/749', 'gsm8k/test/750', 'gsm8k/test/751', 'gsm8k/test/752', 'gsm8k/test/753', 'gsm8k/test/754', 'gsm8k/test/755', 'gsm8k/test/756', 'gsm8k/test/757', 'gsm8k/test/758', 'gsm8k/test/759', 'gsm8k/test/760', 'gsm8k/test/761', 'gsm8k/test/762', 'gsm8k/test/763', 'gsm8k/test/764', 'gsm8k/test/765', 'gsm8k/test/766', 'gsm8k/test/767', 'gsm8k/test/768', 'gsm8k/test/769', 'gsm8k/test/770', 'gsm8k/test/771', 'gsm8k/test/772', 'gsm8k/test/773', 'gsm8k/test/774', 'gsm8k/test/775', 'gsm8k/test/776', 'gsm8k/test/777', 'gsm8k/test/778', 'gsm8k/test/779', 'gsm8k/test/780', 'gsm8k/test/781', 'gsm8k/test/782', 'gsm8k/test/783', 'gsm8k/test/784', 'gsm8k/test/785', 'gsm8k/test/786', 'gsm8k/test/787', 'gsm8k/test/788', 'gsm8k/test/789', 'gsm8k/test/790', 'gsm8k/test/791', 'gsm8k/test/792', 'gsm8k/test/793', 'gsm8k/test/794', 'gsm8k/test/795', 'gsm8k/test/796', 'gsm8k/test/797', 'gsm8k/test/798', 'gsm8k/test/799', 'gsm8k/test/800', 'gsm8k/test/801', 'gsm8k/test/802', 'gsm8k/test/803', 'gsm8k/test/804', 'gsm8k/test/805', 'gsm8k/test/806', 'gsm8k/test/807', 'gsm8k/test/808', 'gsm8k/test/809', 'gsm8k/test/810', 'gsm8k/test/811', 'gsm8k/test/812', 'gsm8k/test/813', 'gsm8k/test/814', 'gsm8k/test/815', 'gsm8k/test/816', 'gsm8k/test/817', 'gsm8k/test/818', 'gsm8k/test/819', 'gsm8k/test/820', 'gsm8k/test/821', 'gsm8k/test/822', 'gsm8k/test/823', 'gsm8k/test/824', 'gsm8k/test/825', 'gsm8k/test/826', 'gsm8k/test/827', 'gsm8k/test/828', 'gsm8k/test/829', 'gsm8k/test/830', 'gsm8k/test/831', 'gsm8k/test/832', 'gsm8k/test/833', 'gsm8k/test/834', 'gsm8k/test/835', 'gsm8k/test/836', 'gsm8k/test/837', 'gsm8k/test/838', 'gsm8k/test/839', 'gsm8k/test/840', 'gsm8k/test/841', 'gsm8k/test/842', 'gsm8k/test/843', 'gsm8k/test/844', 'gsm8k/test/845', 'gsm8k/test/846', 'gsm8k/test/847', 'gsm8k/test/848', 'gsm8k/test/849', 'gsm8k/test/850', 'gsm8k/test/851', 'gsm8k/test/852', 'gsm8k/test/853', 'gsm8k/test/854', 'gsm8k/test/855', 'gsm8k/test/856', 'gsm8k/test/857', 'gsm8k/test/858', 'gsm8k/test/859', 'gsm8k/test/860', 'gsm8k/test/861', 'gsm8k/test/862', 'gsm8k/test/863', 'gsm8k/test/864', 'gsm8k/test/865', 'gsm8k/test/866', 'gsm8k/test/867', 'gsm8k/test/868', 'gsm8k/test/869', 'gsm8k/test/870', 'gsm8k/test/871', 'gsm8k/test/872', 'gsm8k/test/873', 'gsm8k/test/874', 'gsm8k/test/875', 'gsm8k/test/876', 'gsm8k/test/877', 'gsm8k/test/878', 'gsm8k/test/879', 'gsm8k/test/880', 'gsm8k/test/881', 'gsm8k/test/882', 'gsm8k/test/883', 'gsm8k/test/884', 'gsm8k/test/885', 'gsm8k/test/886', 'gsm8k/test/887', 'gsm8k/test/888', 'gsm8k/test/889', 'gsm8k/test/890', 'gsm8k/test/891', 'gsm8k/test/892', 'gsm8k/test/893', 'gsm8k/test/894', 'gsm8k/test/895', 'gsm8k/test/896', 'gsm8k/test/897', 'gsm8k/test/898', 'gsm8k/test/899', 'gsm8k/test/900', 'gsm8k/test/901', 'gsm8k/test/902', 'gsm8k/test/903', 'gsm8k/test/904', 'gsm8k/test/905', 'gsm8k/test/906', 'gsm8k/test/907', 'gsm8k/test/908', 'gsm8k/test/909', 'gsm8k/test/910', 'gsm8k/test/911', 'gsm8k/test/912', 'gsm8k/test/913', 'gsm8k/test/914', 'gsm8k/test/915', 'gsm8k/test/916', 'gsm8k/test/917', 'gsm8k/test/918', 'gsm8k/test/919', 'gsm8k/test/920', 'gsm8k/test/921', 'gsm8k/test/922', 'gsm8k/test/923', 'gsm8k/test/924', 'gsm8k/test/925', 'gsm8k/test/926', 'gsm8k/test/927', 'gsm8k/test/928', 'gsm8k/test/929', 'gsm8k/test/930', 'gsm8k/test/931', 'gsm8k/test/932', 'gsm8k/test/933', 'gsm8k/test/934', 'gsm8k/test/935', 'gsm8k/test/936', 'gsm8k/test/937', 'gsm8k/test/938', 'gsm8k/test/939', 'gsm8k/test/940', 'gsm8k/test/941', 'gsm8k/test/942', 'gsm8k/test/943', 'gsm8k/test/944', 'gsm8k/test/945', 'gsm8k/test/946', 'gsm8k/test/947', 'gsm8k/test/948', 'gsm8k/test/949', 'gsm8k/test/950', 'gsm8k/test/951', 'gsm8k/test/952', 'gsm8k/test/953', 'gsm8k/test/954', 'gsm8k/test/955', 'gsm8k/test/956', 'gsm8k/test/957', 'gsm8k/test/958', 'gsm8k/test/959', 'gsm8k/test/960', 'gsm8k/test/961', 'gsm8k/test/962', 'gsm8k/test/963', 'gsm8k/test/964', 'gsm8k/test/965', 'gsm8k/test/966', 'gsm8k/test/967', 'gsm8k/test/968', 'gsm8k/test/969', 'gsm8k/test/970', 'gsm8k/test/971', 'gsm8k/test/972', 'gsm8k/test/973', 'gsm8k/test/974', 'gsm8k/test/975', 'gsm8k/test/976', 'gsm8k/test/977', 'gsm8k/test/978', 'gsm8k/test/979', 'gsm8k/test/980', 'gsm8k/test/981', 'gsm8k/test/982', 'gsm8k/test/983', 'gsm8k/test/984', 'gsm8k/test/985', 'gsm8k/test/986', 'gsm8k/test/987', 'gsm8k/test/988', 'gsm8k/test/989', 'gsm8k/test/990', 'gsm8k/test/991', 'gsm8k/test/992', 'gsm8k/test/993', 'gsm8k/test/994', 'gsm8k/test/995', 'gsm8k/test/996', 'gsm8k/test/997', 'gsm8k/test/998', 'gsm8k/test/999', 'gsm8k/test/1000', 'gsm8k/test/1001', 'gsm8k/test/1002', 'gsm8k/test/1003', 'gsm8k/test/1004', 'gsm8k/test/1005', 'gsm8k/test/1006', 'gsm8k/test/1007', 'gsm8k/test/1008', 'gsm8k/test/1009', 'gsm8k/test/1010', 'gsm8k/test/1011', 'gsm8k/test/1012', 'gsm8k/test/1013', 'gsm8k/test/1014', 'gsm8k/test/1015', 'gsm8k/test/1016', 'gsm8k/test/1017', 'gsm8k/test/1018', 'gsm8k/test/1019', 'gsm8k/test/1020', 'gsm8k/test/1021', 'gsm8k/test/1022', 'gsm8k/test/1023', 'gsm8k/test/1024', 'gsm8k/test/1025', 'gsm8k/test/1026', 'gsm8k/test/1027', 'gsm8k/test/1028', 'gsm8k/test/1029', 'gsm8k/test/1030', 'gsm8k/test/1031', 'gsm8k/test/1032', 'gsm8k/test/1033', 'gsm8k/test/1034', 'gsm8k/test/1035', 'gsm8k/test/1036', 'gsm8k/test/1037', 'gsm8k/test/1038', 'gsm8k/test/1039', 'gsm8k/test/1040', 'gsm8k/test/1041', 'gsm8k/test/1042', 'gsm8k/test/1043', 'gsm8k/test/1044', 'gsm8k/test/1045', 'gsm8k/test/1046', 'gsm8k/test/1047', 'gsm8k/test/1048', 'gsm8k/test/1049', 'gsm8k/test/1050', 'gsm8k/test/1051', 'gsm8k/test/1052', 'gsm8k/test/1053', 'gsm8k/test/1054', 'gsm8k/test/1055', 'gsm8k/test/1056', 'gsm8k/test/1057', 'gsm8k/test/1058', 'gsm8k/test/1059', 'gsm8k/test/1060', 'gsm8k/test/1061', 'gsm8k/test/1062', 'gsm8k/test/1063', 'gsm8k/test/1064', 'gsm8k/test/1065', 'gsm8k/test/1066', 'gsm8k/test/1067', 'gsm8k/test/1068', 'gsm8k/test/1069', 'gsm8k/test/1070', 'gsm8k/test/1071', 'gsm8k/test/1072', 'gsm8k/test/1073', 'gsm8k/test/1074', 'gsm8k/test/1075', 'gsm8k/test/1076', 'gsm8k/test/1077', 'gsm8k/test/1078', 'gsm8k/test/1079', 'gsm8k/test/1080', 'gsm8k/test/1081', 'gsm8k/test/1082', 'gsm8k/test/1083', 'gsm8k/test/1084', 'gsm8k/test/1085', 'gsm8k/test/1086', 'gsm8k/test/1087', 'gsm8k/test/1088', 'gsm8k/test/1089', 'gsm8k/test/1090', 'gsm8k/test/1091', 'gsm8k/test/1092', 'gsm8k/test/1093', 'gsm8k/test/1094', 'gsm8k/test/1095', 'gsm8k/test/1096', 'gsm8k/test/1097', 'gsm8k/test/1098', 'gsm8k/test/1099', 'gsm8k/test/1100', 'gsm8k/test/1101', 'gsm8k/test/1102', 'gsm8k/test/1103', 'gsm8k/test/1104', 'gsm8k/test/1105', 'gsm8k/test/1106', 'gsm8k/test/1107', 'gsm8k/test/1108', 'gsm8k/test/1109', 'gsm8k/test/1110', 'gsm8k/test/1111', 'gsm8k/test/1112', 'gsm8k/test/1113', 'gsm8k/test/1114', 'gsm8k/test/1115', 'gsm8k/test/1116', 'gsm8k/test/1117', 'gsm8k/test/1118', 'gsm8k/test/1119', 'gsm8k/test/1120', 'gsm8k/test/1121', 'gsm8k/test/1122', 'gsm8k/test/1123', 'gsm8k/test/1124', 'gsm8k/test/1125', 'gsm8k/test/1126', 'gsm8k/test/1127', 'gsm8k/test/1128', 'gsm8k/test/1129', 'gsm8k/test/1130', 'gsm8k/test/1131', 'gsm8k/test/1132', 'gsm8k/test/1133', 'gsm8k/test/1134', 'gsm8k/test/1135', 'gsm8k/test/1136', 'gsm8k/test/1137', 'gsm8k/test/1138', 'gsm8k/test/1139', 'gsm8k/test/1140', 'gsm8k/test/1141', 'gsm8k/test/1142', 'gsm8k/test/1143', 'gsm8k/test/1144', 'gsm8k/test/1145', 'gsm8k/test/1146', 'gsm8k/test/1147', 'gsm8k/test/1148', 'gsm8k/test/1149', 'gsm8k/test/1150', 'gsm8k/test/1151', 'gsm8k/test/1152', 'gsm8k/test/1153', 'gsm8k/test/1154', 'gsm8k/test/1155', 'gsm8k/test/1156', 'gsm8k/test/1157', 'gsm8k/test/1158', 'gsm8k/test/1159', 'gsm8k/test/1160', 'gsm8k/test/1161', 'gsm8k/test/1162', 'gsm8k/test/1163', 'gsm8k/test/1164', 'gsm8k/test/1165', 'gsm8k/test/1166', 'gsm8k/test/1167', 'gsm8k/test/1168', 'gsm8k/test/1169', 'gsm8k/test/1170', 'gsm8k/test/1171', 'gsm8k/test/1172', 'gsm8k/test/1173', 'gsm8k/test/1174', 'gsm8k/test/1175', 'gsm8k/test/1176', 'gsm8k/test/1177', 'gsm8k/test/1178', 'gsm8k/test/1179', 'gsm8k/test/1180', 'gsm8k/test/1181', 'gsm8k/test/1182', 'gsm8k/test/1183', 'gsm8k/test/1184', 'gsm8k/test/1185', 'gsm8k/test/1186', 'gsm8k/test/1187', 'gsm8k/test/1188', 'gsm8k/test/1189', 'gsm8k/test/1190', 'gsm8k/test/1191', 'gsm8k/test/1192', 'gsm8k/test/1193', 'gsm8k/test/1194', 'gsm8k/test/1195', 'gsm8k/test/1196', 'gsm8k/test/1197', 'gsm8k/test/1198', 'gsm8k/test/1199', 'gsm8k/test/1200', 'gsm8k/test/1201', 'gsm8k/test/1202', 'gsm8k/test/1203', 'gsm8k/test/1204', 'gsm8k/test/1205', 'gsm8k/test/1206', 'gsm8k/test/1207', 'gsm8k/test/1208', 'gsm8k/test/1209', 'gsm8k/test/1210', 'gsm8k/test/1211', 'gsm8k/test/1212', 'gsm8k/test/1213', 'gsm8k/test/1214', 'gsm8k/test/1215', 'gsm8k/test/1216', 'gsm8k/test/1217', 'gsm8k/test/1218', 'gsm8k/test/1219', 'gsm8k/test/1220', 'gsm8k/test/1221', 'gsm8k/test/1222', 'gsm8k/test/1223', 'gsm8k/test/1224', 'gsm8k/test/1225', 'gsm8k/test/1226', 'gsm8k/test/1227', 'gsm8k/test/1228', 'gsm8k/test/1229', 'gsm8k/test/1230', 'gsm8k/test/1231', 'gsm8k/test/1232', 'gsm8k/test/1233', 'gsm8k/test/1234', 'gsm8k/test/1235', 'gsm8k/test/1236', 'gsm8k/test/1237', 'gsm8k/test/1238', 'gsm8k/test/1239', 'gsm8k/test/1240', 'gsm8k/test/1241', 'gsm8k/test/1242', 'gsm8k/test/1243', 'gsm8k/test/1244', 'gsm8k/test/1245', 'gsm8k/test/1246', 'gsm8k/test/1247', 'gsm8k/test/1248', 'gsm8k/test/1249', 'gsm8k/test/1250', 'gsm8k/test/1251', 'gsm8k/test/1252', 'gsm8k/test/1253', 'gsm8k/test/1254', 'gsm8k/test/1255', 'gsm8k/test/1256', 'gsm8k/test/1257', 'gsm8k/test/1258', 'gsm8k/test/1259', 'gsm8k/test/1260', 'gsm8k/test/1261', 'gsm8k/test/1262', 'gsm8k/test/1263', 'gsm8k/test/1264', 'gsm8k/test/1265', 'gsm8k/test/1266', 'gsm8k/test/1267', 'gsm8k/test/1268', 'gsm8k/test/1269', 'gsm8k/test/1270', 'gsm8k/test/1271', 'gsm8k/test/1272', 'gsm8k/test/1273', 'gsm8k/test/1274', 'gsm8k/test/1275', 'gsm8k/test/1276', 'gsm8k/test/1277', 'gsm8k/test/1278', 'gsm8k/test/1279', 'gsm8k/test/1280', 'gsm8k/test/1281', 'gsm8k/test/1282', 'gsm8k/test/1283', 'gsm8k/test/1284', 'gsm8k/test/1285', 'gsm8k/test/1286', 'gsm8k/test/1287', 'gsm8k/test/1288', 'gsm8k/test/1289', 'gsm8k/test/1290', 'gsm8k/test/1291', 'gsm8k/test/1292', 'gsm8k/test/1293', 'gsm8k/test/1294', 'gsm8k/test/1295', 'gsm8k/test/1296', 'gsm8k/test/1297', 'gsm8k/test/1298', 'gsm8k/test/1299', 'gsm8k/test/1300', 'gsm8k/test/1301', 'gsm8k/test/1302', 'gsm8k/test/1303', 'gsm8k/test/1304', 'gsm8k/test/1305', 'gsm8k/test/1306', 'gsm8k/test/1307', 'gsm8k/test/1308', 'gsm8k/test/1309', 'gsm8k/test/1310', 'gsm8k/test/1311', 'gsm8k/test/1312', 'gsm8k/test/1313', 'gsm8k/test/1314', 'gsm8k/test/1315', 'gsm8k/test/1316', 'gsm8k/test/1317', 'gsm8k/test/1318']}}, 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052': {'code_sha256': '0b46931e371b503139a3dc29febd95819f24059f6bab50e477354666bfad5036', 'prompt_template': {'renderer': 'role_colon', 'tokenizer': 'Qwen/Qwen2.5-3B', 'convo': 'MathEnv.standard_fewshot_prefix() + [user: question + Gsm8kDataset.question_suffix()]', 'question_suffix': ' Provide a numerical answer without units, written inside \\boxed{}.', 'build': 'renderer.build_generation_prompt(convo)'}, 'decoding': {'temperature': 0.0, 'max_tokens': 512, 'stop': 'renderer.get_stop_sequences() (role_colon)'}, 'parser': {'code': 'eval_gsm8k.parse/correct', 'rule': "last number in last \\boxed{} (else last number in text), strip ',' '$', float; correct iff finite and == float(reference '####' answer)"}, 'items': {'ids': ['gsm8k/test/0', 'gsm8k/test/1', 'gsm8k/test/2', 'gsm8k/test/3', 'gsm8k/test/4', 'gsm8k/test/5', 'gsm8k/test/6', 'gsm8k/test/7', 'gsm8k/test/8', 'gsm8k/test/9', 'gsm8k/test/10', 'gsm8k/test/11', 'gsm8k/test/12', 'gsm8k/test/13', 'gsm8k/test/14', 'gsm8k/test/15', 'gsm8k/test/16', 'gsm8k/test/17', 'gsm8k/test/18', 'gsm8k/test/19', 'gsm8k/test/20', 'gsm8k/test/21', 'gsm8k/test/22', 'gsm8k/test/23', 'gsm8k/test/24', 'gsm8k/test/25', 'gsm8k/test/26', 'gsm8k/test/27', 'gsm8k/test/28', 'gsm8k/test/29', 'gsm8k/test/30', 'gsm8k/test/31', 'gsm8k/test/32', 'gsm8k/test/33', 'gsm8k/test/34', 'gsm8k/test/35', 'gsm8k/test/36', 'gsm8k/test/37', 'gsm8k/test/38', 'gsm8k/test/39', 'gsm8k/test/40', 'gsm8k/test/41', 'gsm8k/test/42', 'gsm8k/test/43', 'gsm8k/test/44', 'gsm8k/test/45', 'gsm8k/test/46', 'gsm8k/test/47', 'gsm8k/test/48', 'gsm8k/test/49', 'gsm8k/test/50', 'gsm8k/test/51', 'gsm8k/test/52', 'gsm8k/test/53', 'gsm8k/test/54', 'gsm8k/test/55', 'gsm8k/test/56', 'gsm8k/test/57', 'gsm8k/test/58', 'gsm8k/test/59', 'gsm8k/test/60', 'gsm8k/test/61', 'gsm8k/test/62', 'gsm8k/test/63', 'gsm8k/test/64', 'gsm8k/test/65', 'gsm8k/test/66', 'gsm8k/test/67', 'gsm8k/test/68', 'gsm8k/test/69', 'gsm8k/test/70', 'gsm8k/test/71', 'gsm8k/test/72', 'gsm8k/test/73', 'gsm8k/test/74', 'gsm8k/test/75', 'gsm8k/test/76', 'gsm8k/test/77', 'gsm8k/test/78', 'gsm8k/test/79', 'gsm8k/test/80', 'gsm8k/test/81', 'gsm8k/test/82', 'gsm8k/test/83', 'gsm8k/test/84', 'gsm8k/test/85', 'gsm8k/test/86', 'gsm8k/test/87', 'gsm8k/test/88', 'gsm8k/test/89', 'gsm8k/test/90', 'gsm8k/test/91', 'gsm8k/test/92', 'gsm8k/test/93', 'gsm8k/test/94', 'gsm8k/test/95', 'gsm8k/test/96', 'gsm8k/test/97', 'gsm8k/test/98', 'gsm8k/test/99', 'gsm8k/test/100', 'gsm8k/test/101', 'gsm8k/test/102', 'gsm8k/test/103', 'gsm8k/test/104', 'gsm8k/test/105', 'gsm8k/test/106', 'gsm8k/test/107', 'gsm8k/test/108', 'gsm8k/test/109', 'gsm8k/test/110', 'gsm8k/test/111', 'gsm8k/test/112', 'gsm8k/test/113', 'gsm8k/test/114', 'gsm8k/test/115', 'gsm8k/test/116', 'gsm8k/test/117', 'gsm8k/test/118', 'gsm8k/test/119', 'gsm8k/test/120', 'gsm8k/test/121', 'gsm8k/test/122', 'gsm8k/test/123', 'gsm8k/test/124', 'gsm8k/test/125', 'gsm8k/test/126', 'gsm8k/test/127', 'gsm8k/test/128', 'gsm8k/test/129', 'gsm8k/test/130', 'gsm8k/test/131', 'gsm8k/test/132', 'gsm8k/test/133', 'gsm8k/test/134', 'gsm8k/test/135', 'gsm8k/test/136', 'gsm8k/test/137', 'gsm8k/test/138', 'gsm8k/test/139', 'gsm8k/test/140', 'gsm8k/test/141', 'gsm8k/test/142', 'gsm8k/test/143', 'gsm8k/test/144', 'gsm8k/test/145', 'gsm8k/test/146', 'gsm8k/test/147', 'gsm8k/test/148', 'gsm8k/test/149', 'gsm8k/test/150', 'gsm8k/test/151', 'gsm8k/test/152', 'gsm8k/test/153', 'gsm8k/test/154', 'gsm8k/test/155', 'gsm8k/test/156', 'gsm8k/test/157', 'gsm8k/test/158', 'gsm8k/test/159', 'gsm8k/test/160', 'gsm8k/test/161', 'gsm8k/test/162', 'gsm8k/test/163', 'gsm8k/test/164', 'gsm8k/test/165', 'gsm8k/test/166', 'gsm8k/test/167', 'gsm8k/test/168', 'gsm8k/test/169', 'gsm8k/test/170', 'gsm8k/test/171', 'gsm8k/test/172', 'gsm8k/test/173', 'gsm8k/test/174', 'gsm8k/test/175', 'gsm8k/test/176', 'gsm8k/test/177', 'gsm8k/test/178', 'gsm8k/test/179', 'gsm8k/test/180', 'gsm8k/test/181', 'gsm8k/test/182', 'gsm8k/test/183', 'gsm8k/test/184', 'gsm8k/test/185', 'gsm8k/test/186', 'gsm8k/test/187', 'gsm8k/test/188', 'gsm8k/test/189', 'gsm8k/test/190', 'gsm8k/test/191', 'gsm8k/test/192', 'gsm8k/test/193', 'gsm8k/test/194', 'gsm8k/test/195', 'gsm8k/test/196', 'gsm8k/test/197', 'gsm8k/test/198', 'gsm8k/test/199']}}}, 'scores': [{'arm': 'base', 'metric': 'selection_accuracy', 'value': 0.71, 'evaluator_id': 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052', 'seed': 0, 'predictions_ref': 'pred_selection_base.jsonl'}, {'arm': 'rank1_lr1e-3', 'metric': 'selection_accuracy', 'value': 0.0, 'evaluator_id': 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052', 'seed': 0, 'predictions_ref': 'pred_selection_rank1_lr1e-3.jsonl'}, {'arm': 'rank1_lr1e-4', 'metric': 'selection_accuracy', 'value': 0.82, 'evaluator_id': 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052', 'seed': 0, 'predictions_ref': 'pred_selection_rank1_lr1e-4.jsonl'}, {'arm': 'rank1_lr3e-4', 'metric': 'selection_accuracy', 'value': 0.82, 'evaluator_id': 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052', 'seed': 0, 'predictions_ref': 'pred_selection_rank1_lr3e-4.jsonl'}, {'arm': 'rank1_lr3e-5', 'metric': 'selection_accuracy', 'value': 0.82, 'evaluator_id': 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052', 'seed': 0, 'predictions_ref': 'pred_selection_rank1_lr3e-5.jsonl'}, {'arm': 'rank32_lr1e-3', 'metric': 'selection_accuracy', 'value': 0.0, 'evaluator_id': 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052', 'seed': 0, 'predictions_ref': 'pred_selection_rank32_lr1e-3.jsonl'}, {'arm': 'rank32_lr1e-4', 'metric': 'selection_accuracy', 'value': 0.815, 'evaluator_id': 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052', 'seed': 0, 'predictions_ref': 'pred_selection_rank32_lr1e-4.jsonl'}, {'arm': 'rank32_lr3e-4', 'metric': 'selection_accuracy', 'value': 0.84, 'evaluator_id': 'd8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052', 'seed': 0, 'predictions_ref': 'pred_selection_rank32_lr3e-4.jsonl'}, {'arm': 'treatment', 'metric': 'accuracy', 'value': 0.8239499553172476, 'evaluator_id': 'f1d45345e6712abaa28f75c82cbb2800cdb101b45ec44e5d87e02d7ff47b3023', 'seed': 0, 'predictions_ref': 'pred_confirmation_rank1_lr3e-5.jsonl', 'checkpoint_arm': 'rank1_lr3e-5'}, {'arm': 'baseline', 'metric': 'accuracy', 'value': 0.8105451295799821, 'evaluator_id': 'f1d45345e6712abaa28f75c82cbb2800cdb101b45ec44e5d87e02d7ff47b3023', 'seed': 0, 'predictions_ref': 'pred_confirmation_rank32_lr3e-4.jsonl', 'checkpoint_arm': 'rank32_lr3e-4'}, {'arm': 'base', 'metric': 'accuracy', 'value': 0.7345844504021448, 'evaluator_id': 'f1d45345e6712abaa28f75c82cbb2800cdb101b45ec44e5d87e02d7ff47b3023', 'seed': 0, 'predictions_ref': 'pred_confirmation_base.jsonl', 'checkpoint_arm': 'base'}], 'comparisons': [{'metric': 'selection_accuracy', 'arms': ['base', 'rank1_lr1e-3', 'rank1_lr1e-4', 'rank1_lr3e-4', 'rank1_lr3e-5', 'rank32_lr1e-3', 'rank32_lr1e-4', 'rank32_lr3e-4']}, {'metric': 'accuracy', 'arms': ['treatment', 'baseline']}, {'metric': 'accuracy', 'arms': ['treatment', 'baseline', 'base']}]}

evaluator_validation

{'status': 'comparable', 'issues': [], 'comparisons': [{'metric': 'accuracy', 'status': 'comparable', 'arms': ['base', 'baseline', 'treatment'], 'evaluator_ids': ['f1d45345e6712abaa28f75c82cbb2800cdb101b45ec44e5d87e02d7ff47b3023'], 'missing_arms': []}, {'metric': 'selection_accuracy', 'status': 'comparable', 'arms': ['base', 'rank1_lr1e-3', 'rank1_lr1e-4', 'rank1_lr3e-4', 'rank1_lr3e-5', 'rank32_lr1e-3', 'rank32_lr1e-4', 'rank32_lr3e-4'], 'evaluator_ids': ['d8a177a50b10b110e46f15106fe16e7ea14ec007c9956519067cec4b67256052'], 'missing_arms': []}]}

feasibility

{'status': 'unknown', 'problems': ['Re-projection required: lora_rank changed or was not recorded', 'Re-projection required: dataset changed or was not recorded', 'Re-projection required: steps changed or was not recorded', 'Re-projection required: batch_size changed or was not recorded', 'Re-projection required: max_seq_len changed or was not recorded', 'Re-projection required: arms changed or was not recorded', 'Re-projection required: n_examples changed or was not recorded', 'Re-projection required: n_eval changed or was not recorded'], 'wall_seconds': None, 'gpu_hours': None, 'known_gpu_hours': 0, 'phases': {'setup': {'wall_seconds': 31.1, 'planned_units': 1, 'planned_samples': 0.0, 'gpu_hours': None}, 'training': {'wall_seconds': 2900.0, 'planned_units': 320, 'planned_samples': 20480.0, 'gpu_hours': None}, 'evaluation': {'wall_seconds': 391.40000000000003, 'planned_units': 4957, 'planned_samples': 4957.0, 'gpu_hours': None}, 'recovery': {'wall_seconds': 1200, 'gpu_hours': None}}, 'generation_counts': None, 'generation_applicability': {'check': 'generation_audit', 'status': 'not_applicable', 'enforce': False, 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'features': {'generated_training_data': False}}, 'record': {'config': {'base_model': 'Qwen/Qwen2.5-3B', 'arms': ['rank1_lr3e-4', 'rank32_lr3e-4'], 'steps': 3, 'group_size': 8, 'groups_per_batch': 8, 'max_tokens': 512, 'n_eval': 200}, 'effective_config': {'base_model': 'Qwen/Qwen2.5-3B', 'arms': ['rank1_lr3e-5', 'rank1_lr1e-4', 'rank1_lr3e-4', 'rank1_lr1e-3', 'rank32_lr1e-4', 'rank32_lr3e-4', 'rank32_lr1e-3'], 'steps': 40, 'batch_size': 64, 'groups_per_batch': 8, 'group_size': 8, 'max_tokens': 512, 'loss_fn': 'importance_sampling', 'renderer': 'role_colon', 'n_eval': {'selection': 1600, 'confirmation': 3357}}, 'primary_endpoint': 'gsm8k_test_confirmation_exact_match_accuracy', 'source': 'training-metrics.json, eval-scores-and-selection.json, probe-results.json, logs-excerpts.json', 'measurements': {'setup': {'seconds': 31.1, 'units': 1, 'planned_units': 1, 'samples': 0, 'gpu_count': None, 'allocation_source': None}, 'training': {'seconds': 2900, 'units': 320, 'planned_units': 320, 'samples': 20480, 'gpu_count': None, 'allocation_source': None}, 'evaluation': {'seconds': 391.4, 'units': 4957, 'planned_units': 4957, 'samples': 4957, 'gpu_count': None, 'allocation_source': None}}, 'generation': None, 'recovery': {'seconds': 1200, 'gpu_count': None, 'allocation_source': None}}, 'recorded_at': 1793815440.1543531}

feasibility_reported_verdict

refuted

feasibility_required

True

generation_audit_validation

{'status': 'not_applicable', 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'applicability': {'check': 'generation_audit', 'status': 'not_applicable', 'enforce': False, 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'features': {'generated_training_data': False}}}

initialization_provenance

{'version': 1, 'arms': [{'name': 'rank1_lr1e-4', 'initialization': {'base_checkpoint': 'Qwen/Qwen2.5-3B@3aab1f1954e9cc14eb9509a215f9e5ca08227a9b', 'source': 'server_response', 'adapter_sha256': None, 'adapter_unavailable_reason': 'not attempted: save_weights_external returned 404 Not Found on shared Spindle in the preflight probe', 'server_model_id': '36d478cb73c0d4af71be9e84880ce6fc', 'server_lora_rank': 1, 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}, 'restore_count': 0, 'optimizer_checks': [{'after_restore': 0, 'actual': 'adamw', 'source': 'server_response', 'actual_config': {}, 'client_sent_adam_params_first_step': {'learning_rate': 0.0001, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'evidence_ref': 'init-provenance-bundle.json', 'before_next_update': True}], 'initial_output': {'available': True, 'batch_sha256': 'bd6e130ab84c8977aea60bc4571297ae3a3a090eb9d9463fcbc3439214bfcb3e', 'output_sha256': 'c3ff0be48f064881c30bbeb9c7454d22ad3f4749393ff56c94ff36c6bfb96731', 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}}, {'name': 'rank1_lr3e-4', 'initialization': {'base_checkpoint': 'Qwen/Qwen2.5-3B@3aab1f1954e9cc14eb9509a215f9e5ca08227a9b', 'source': 'server_response', 'adapter_sha256': None, 'adapter_unavailable_reason': 'not attempted: save_weights_external returned 404 Not Found on shared Spindle in the preflight probe', 'server_model_id': '1293cfb24019e3fc1998a35d984bddf0', 'server_lora_rank': 1, 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}, 'restore_count': 0, 'optimizer_checks': [{'after_restore': 0, 'actual': 'adamw', 'source': 'server_response', 'actual_config': {}, 'client_sent_adam_params_first_step': {'learning_rate': 0.0003, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'evidence_ref': 'init-provenance-bundle.json', 'before_next_update': True}], 'initial_output': {'available': True, 'batch_sha256': 'bd6e130ab84c8977aea60bc4571297ae3a3a090eb9d9463fcbc3439214bfcb3e', 'output_sha256': 'c3ff0be48f064881c30bbeb9c7454d22ad3f4749393ff56c94ff36c6bfb96731', 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}}, {'name': 'rank1_lr1e-3', 'initialization': {'base_checkpoint': 'Qwen/Qwen2.5-3B@3aab1f1954e9cc14eb9509a215f9e5ca08227a9b', 'source': 'server_response', 'adapter_sha256': None, 'adapter_unavailable_reason': 'not attempted: save_weights_external returned 404 Not Found on shared Spindle in the preflight probe', 'server_model_id': 'b1503a4ce8731de97e8dc361cde0a152', 'server_lora_rank': 1, 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}, 'restore_count': 0, 'optimizer_checks': [{'after_restore': 0, 'actual': 'adamw', 'source': 'server_response', 'actual_config': {}, 'client_sent_adam_params_first_step': {'learning_rate': 0.001, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'evidence_ref': 'init-provenance-bundle.json', 'before_next_update': True}], 'initial_output': {'available': True, 'batch_sha256': 'bd6e130ab84c8977aea60bc4571297ae3a3a090eb9d9463fcbc3439214bfcb3e', 'output_sha256': 'c3ff0be48f064881c30bbeb9c7454d22ad3f4749393ff56c94ff36c6bfb96731', 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}}, {'name': 'rank32_lr1e-4', 'initialization': {'base_checkpoint': 'Qwen/Qwen2.5-3B@3aab1f1954e9cc14eb9509a215f9e5ca08227a9b', 'source': 'server_response', 'adapter_sha256': None, 'adapter_unavailable_reason': 'not attempted: save_weights_external returned 404 Not Found on shared Spindle in the preflight probe', 'server_model_id': '03c5f3745726df7ccc2371ae19c922c0', 'server_lora_rank': 32, 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}, 'restore_count': 1, 'optimizer_checks': [{'after_restore': 0, 'actual': 'adamw', 'source': 'server_response', 'actual_config': {}, 'client_sent_adam_params_first_step': {'learning_rate': 0.0001, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'evidence_ref': 'init-provenance-bundle.json', 'before_next_update': True}, {'after_restore': 1, 'actual': 'adamw', 'source': 'server_response', 'actual_config': {}, 'client_sent_adam_params_first_step': {'learning_rate': 0.0001, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'evidence_ref': 'init-provenance-bundle.json', 'before_next_update': True}], 'initial_output': {'available': True, 'batch_sha256': 'bd6e130ab84c8977aea60bc4571297ae3a3a090eb9d9463fcbc3439214bfcb3e', 'output_sha256': 'c3ff0be48f064881c30bbeb9c7454d22ad3f4749393ff56c94ff36c6bfb96731', 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}, 'restores': [{'after_restore': 1, 'reason': '410 model_lost (shared Spindle engine terminated) during first attempt', 'restored_from': 'tinker://03c5f3745726df7ccc2371ae19c922c0/weights/000020', 'method': 'create_training_client_from_state_with_optimizer (stock recipe resume)', 'server_lora_rank': 32, 'server_model_id': '4d7ec04e-f22a-4ee5-9c17-64cb108c82e2:train:0', 'fixed_batch_output_sha256_after_restore': '3574f578243683263ee1e2c0aaca92cdc5a5af30e47429ec1d7bd51415555fba', 'evidence_ref': 'init-provenance-bundle.json'}]}, {'name': 'rank32_lr3e-4', 'initialization': {'base_checkpoint': 'Qwen/Qwen2.5-3B@3aab1f1954e9cc14eb9509a215f9e5ca08227a9b', 'source': 'server_response', 'adapter_sha256': None, 'adapter_unavailable_reason': 'not attempted: save_weights_external returned 404 Not Found on shared Spindle in the preflight probe', 'server_model_id': '11d230bfdd99bd73e688b09f9ae51649', 'server_lora_rank': 32, 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}, 'restore_count': 1, 'optimizer_checks': [{'after_restore': 0, 'actual': 'adamw', 'source': 'server_response', 'actual_config': {}, 'client_sent_adam_params_first_step': {'learning_rate': 0.0003, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'evidence_ref': 'init-provenance-bundle.json', 'before_next_update': True}, {'after_restore': 1, 'actual': 'adamw', 'source': 'server_response', 'actual_config': {}, 'client_sent_adam_params_first_step': {'learning_rate': 0.0003, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'evidence_ref': 'init-provenance-bundle.json', 'before_next_update': True}], 'initial_output': {'available': True, 'batch_sha256': 'bd6e130ab84c8977aea60bc4571297ae3a3a090eb9d9463fcbc3439214bfcb3e', 'output_sha256': 'c3ff0be48f064881c30bbeb9c7454d22ad3f4749393ff56c94ff36c6bfb96731', 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}, 'restores': [{'after_restore': 1, 'reason': '410 model_lost (shared Spindle engine terminated) during first attempt', 'restored_from': 'tinker://11d230bfdd99bd73e688b09f9ae51649/weights/000020', 'method': 'create_training_client_from_state_with_optimizer (stock recipe resume)', 'server_lora_rank': 32, 'server_model_id': '2aa88565-ba3e-4d6b-abc3-c0f9777792da:train:0', 'fixed_batch_output_sha256_after_restore': '353dacc564d797bf459ab22cffd2b7e992d68962afd1b7161c5ab956aa647458', 'evidence_ref': 'init-provenance-bundle.json'}]}, {'name': 'rank32_lr1e-3', 'initialization': {'base_checkpoint': 'Qwen/Qwen2.5-3B@3aab1f1954e9cc14eb9509a215f9e5ca08227a9b', 'source': 'server_response', 'adapter_sha256': None, 'adapter_unavailable_reason': 'not attempted: save_weights_external returned 404 Not Found on shared Spindle in the preflight probe', 'server_model_id': 'fa768cbf5c7fd3a1554d3f4319b2478f', 'server_lora_rank': 32, 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}, 'restore_count': 0, 'optimizer_checks': [{'after_restore': 0, 'actual': 'adamw', 'source': 'server_response', 'actual_config': {}, 'client_sent_adam_params_first_step': {'learning_rate': 0.001, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'evidence_ref': 'init-provenance-bundle.json', 'before_next_update': True}], 'initial_output': {'available': True, 'batch_sha256': 'bd6e130ab84c8977aea60bc4571297ae3a3a090eb9d9463fcbc3439214bfcb3e', 'output_sha256': 'c3ff0be48f064881c30bbeb9c7454d22ad3f4749393ff56c94ff36c6bfb96731', 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}}, {'name': 'rank1_lr3e-5', 'initialization': {'base_checkpoint': 'Qwen/Qwen2.5-3B@3aab1f1954e9cc14eb9509a215f9e5ca08227a9b', 'source': 'server_response', 'adapter_sha256': None, 'adapter_unavailable_reason': 'not attempted: save_weights_external returned 404 Not Found on shared Spindle in the preflight probe', 'server_model_id': '080365acb1f7ca92af4613520ce33213', 'server_lora_rank': 1, 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}, 'restore_count': 0, 'optimizer_checks': [{'after_restore': 0, 'actual': 'adamw', 'source': 'server_response', 'actual_config': {}, 'client_sent_adam_params_first_step': {'learning_rate': 3e-05, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'evidence_ref': 'init-provenance-bundle.json', 'before_next_update': True}], 'initial_output': {'available': True, 'batch_sha256': 'bd6e130ab84c8977aea60bc4571297ae3a3a090eb9d9463fcbc3439214bfcb3e', 'output_sha256': 'c3ff0be48f064881c30bbeb9c7454d22ad3f4749393ff56c94ff36c6bfb96731', 'evidence_ref': 'init-provenance-bundle.json', 'before_first_update': True}}]}

launch

{'experiment_id': 'exp_db52ed42bb', 'context_status': 'snapshotted', 'superseded': False, 'mode': 'ordinary', 'purpose': 'Execute the selected saved experiment version.'}

launch_instructions

{'name': 'launch-instructions-9671ae2b3b79.md', 'sha256': '9671ae2b3b79c49d342b16d2da266b23771992ff92215779aca6c67cf91693e6', 'url': 'https://rx.modal.dev/api/runs/run_40338c73ff/evidence/9671ae2b3b79c49d342b16d2da266b23771992ff92215779aca6c67cf91693e6'}

reported_verdict

refuted

restoration_validation

{'status': 'unverified', 'required': True, 'checks': [{'check': 'rank1_lr1e-4.initialization', 'status': 'unverified', 'reason': 'rank1_lr1e-4: need a measured adapter hash or immutable full initialization checkpoint with an observed successful load.'}, {'check': 'rank1_lr1e-4.optimizer.0', 'status': 'unverified', 'reason': "rank1_lr1e-4: optimizer adamw after restore 0; configuration unknown: ['learning_rate', 'beta1', 'beta2', 'eps']."}, {'check': 'rank1_lr1e-4.output', 'status': 'passed', 'reason': 'rank1_lr1e-4: fixed-batch full-output fingerprint recorded.'}, {'check': 'rank1_lr3e-4.initialization', 'status': 'unverified', 'reason': 'rank1_lr3e-4: need a measured adapter hash or immutable full initialization checkpoint with an observed successful load.'}, {'check': 'rank1_lr3e-4.optimizer.0', 'status': 'unverified', 'reason': "rank1_lr3e-4: optimizer adamw after restore 0; configuration unknown: ['learning_rate', 'beta1', 'beta2', 'eps']."}, {'check': 'rank1_lr3e-4.output', 'status': 'passed', 'reason': 'rank1_lr3e-4: fixed-batch full-output fingerprint recorded.'}, {'check': 'rank1_lr1e-3.initialization', 'status': 'unverified', 'reason': 'rank1_lr1e-3: need a measured adapter hash or immutable full initialization checkpoint with an observed successful load.'}, {'check': 'rank1_lr1e-3.optimizer.0', 'status': 'unverified', 'reason': "rank1_lr1e-3: optimizer adamw after restore 0; configuration unknown: ['learning_rate', 'beta1', 'beta2', 'eps']."}, {'check': 'rank1_lr1e-3.output', 'status': 'passed', 'reason': 'rank1_lr1e-3: fixed-batch full-output fingerprint recorded.'}, {'check': 'rank32_lr1e-4.initialization', 'status': 'unverified', 'reason': 'rank32_lr1e-4: need a measured adapter hash or immutable full initialization checkpoint with an observed successful load.'}, {'check': 'rank32_lr1e-4.optimizer.0', 'status': 'unverified', 'reason': "rank32_lr1e-4: optimizer adamw after restore 0; configuration unknown: ['learning_rate', 'beta1', 'beta2', 'eps']."}, {'check': 'rank32_lr1e-4.optimizer.1', 'status': 'unverified', 'reason': "rank32_lr1e-4: optimizer adamw after restore 1; configuration unknown: ['learning_rate', 'beta1', 'beta2', 'eps']."}, {'check': 'rank32_lr1e-4.output', 'status': 'passed', 'reason': 'rank32_lr1e-4: fixed-batch full-output fingerprint recorded.'}, {'check': 'rank32_lr3e-4.initialization', 'status': 'unverified', 'reason': 'rank32_lr3e-4: need a measured adapter hash or immutable full initialization checkpoint with an observed successful load.'}, {'check': 'rank32_lr3e-4.optimizer.0', 'status': 'unverified', 'reason': "rank32_lr3e-4: optimizer adamw after restore 0; configuration unknown: ['learning_rate', 'beta1', 'beta2', 'eps']."}, {'check': 'rank32_lr3e-4.optimizer.1', 'status': 'unverified', 'reason': "rank32_lr3e-4: optimizer adamw after restore 1; configuration unknown: ['learning_rate', 'beta1', 'beta2', 'eps']."}, {'check': 'rank32_lr3e-4.output', 'status': 'passed', 'reason': 'rank32_lr3e-4: fixed-batch full-output fingerprint recorded.'}, {'check': 'rank32_lr1e-3.initialization', 'status': 'unverified', 'reason': 'rank32_lr1e-3: need a measured adapter hash or immutable full initialization checkpoint with an observed successful load.'}, {'check': 'rank32_lr1e-3.optimizer.0', 'status': 'unverified', 'reason': "rank32_lr1e-3: optimizer adamw after restore 0; configuration unknown: ['learning_rate', 'beta1', 'beta2', 'eps']."}, {'check': 'rank32_lr1e-3.output', 'status': 'passed', 'reason': 'rank32_lr1e-3: fixed-batch full-output fingerprint recorded.'}], 'reason': 'rank1_lr1e-4: need a measured adapter hash or immutable full initialization checkpoint with an observed successful load.', 'applicability': 'required'}

slack_started

True

verdict_report

{'effect': 0.013404825737265424, 'margin': -0.09, 'lower': -0.0677935209192818, 'upper': 0.09460317239381262, 'seeds': 1, 'confidence': 'weak', 'reasoning': "The statistical criterion passes: the tuned rank-1 minus tuned rank-32 paired Hoeffding 95% CI over 1119 confirmation items is [-0.068, +0.095], so its lower bound clears the -0.09 margin, and both selected arms beat base by at least 0.05 (+0.089 and +0.076). The spec's success criterion also says a selected LR on a grid edge that cannot be extended is inconclusive. Rank-1's original pick, 1e-4 (a tie with 3e-4 at 0.82), was on the edge; the one allowed extension, 3e-5, also scored 0.82, so the lower-LR tie rule picked 3e-5, again an edge. Because the criterion is not met as written, this is recorded as not supported (weak) rather than supported. Confidence is further limited by one seed, by initialization/optimizer-config provenance that shared Spindle leaves unverified (no adapter download; get_info exposes only optimizer type adamw), and by the spec's string-range item IDs, which make the shared decision validator reject the adaptive-selection plan (with explicit ID lists the same observations assess as passed). LR 1e-3 collapsed both ranks to 0.0.", 'grid_edge': True}