🔬Modal Research Exchange

{
  "activation_gate": {
    "applicable": true,
    "checks": [
      {
        "max": 1,
        "measures": "rollouts per prompt actually sampled in the GS1 arm",
        "metric": "gs1_executed_group_size",
        "min": 1
      },
      {
        "max": 64,
        "measures": "total rollouts per optimizer step actually executed in each arm",
        "metric": "rollouts_per_step_both_arms",
        "min": 64
      },
      {
        "max": 40,
        "measures": "median over i of 1/sum_j w_ij^2. It must be well below the uniform value of 63 and above degenerate single-neighbor weighting",
        "metric": "gs1_effective_contributors_median",
        "min": 2
      },
      {
        "measures": "mean |b_i - uniform LOO mean|. If ~0, the similarity weighting does nothing",
        "metric": "gs1_mean_abs_baseline_minus_uniform_loo",
        "min": 0.02
      },
      {
        "max": 0.95,
        "measures": "mean train reward in pilot steps. Mid-range reward is needed so that advantages are nonzero",
        "metric": "batch_mean_reward_pilot",
        "min": 0.1
      },
      {
        "measures": "model name reported by the training client",
        "metric": "executed_base_model",
        "one_of": [
          "Qwen/Qwen3.5-2B"
        ]
      }
    ],
    "reason": "These checks confirm that the GS1 baseline is actually prompt-weighted rather than a uniform batch mean, that GS1 has learning signal where GRPO groups would be all-equal, and that the executed configs match the plan. They are measured on a 3-step pilot of each arm."
  },
  "arms": [
    {
      "advantage_mode": "grpo",
      "group_size": 8,
      "groups_per_batch": 8,
      "lr": 4e-05,
      "name": "grpo_gs8_seed0",
      "seed": 0,
      "steps": 40
    },
    {
      "advantage_mode": "gs1",
      "group_size": 1,
      "groups_per_batch": 64,
      "lr": 4e-05,
      "name": "gs1_seed0",
      "seed": 0,
      "similarity": "tfidf_prompt_cosine",
      "steps": 40,
      "tau": 0.1
    }
  ],
  "base_model": "Qwen/Qwen3.5-2B",
  "baseline": "GRPO with group_size=8 and groups_per_batch=8 (64 rollouts/step) and a group-mean advantage. It runs through the same script with advantage_mode=grpo and shares model, LoRA init, lr, max_tokens, steps, reward, renderer and eval.",
  "batch_size": 64,
  "budget": {
    "gpu_hours": 2.0,
    "max_wall_minutes": 150
  },
  "command": "# Custom part: the advantage computation only. Everything else is the stock cookbook RL loop and loss.\n# Stock reference for the baseline arm:\n#   python -m tinker_cookbook.recipes.math_rl.train model_name=Qwen/Qwen3.5-2B base_url=$TINKER_BASE_URL env=gsm8k group_size=8 groups_per_batch=8 learning_rate=4e-5 lora_rank=32 max_tokens=512 max_steps=40 log_path=/tmp/rx-grpo\n# Both arms through one script so they differ only in advantage_mode:\npython gs1_vs_grpo.py --base_url $TINKER_BASE_URL --model Qwen/Qwen3.5-2B --lora_rank 32 --lr 4e-5 --max_tokens 512 --steps 40 --rollouts_per_step 64 --seed 0 --advantage_mode {grpo|gs1} --tau 0.1 --log_path /tmp/rx-{arm}\n# gs1_vs_grpo.py:\n# - client = tinker.ServiceClient(base_url=os.environ['TINKER_BASE_URL'])\n# - tc = client.create_lora_training_client(base_model=..., rank=32)\n# - per step: sample via tc.save_weights_and_get_sampling_client()\n#   - grpo: 8 prompts x 8 samples\n#   - gs1: 64 prompts x 1 sample\n# - reward from the cookbook gsm8k env\n# - advantages:\n#   - grpo: r - group_mean\n#   - gs1: r_i - sum_{j!=i} softmax_j(cos(tfidf_i, tfidf_j)/tau) r_j  [CUSTOM]\n# - tc.forward_backward(datums, loss_fn='importance_sampling'); tc.optim_step(tinker.AdamParams(learning_rate=4e-5))\n# - eval: greedy sample on all 1319 GSM8K test items from the step-40 sampler, numeric exact match, per-item results saved for pairing",
  "correctness_preflight": {
    "paths": [
      {
        "baseline_revision": "UNRESOLVED: pin tinker-cookbook commit SHA (rl/train.py, importance_sampling loss) before running",
        "id": "gs1_advantage_rl_loss",
        "kind": "loss",
        "reference_revision": "UNRESOLVED: pin trajectory.ai 'You Only Rollout Once' reference code commit SHA, or a reviewed independent float64 reference implementation at a pinned SHA",
        "tolerances": {
          "baseline_gradient": {
            "atol": 1e-06,
            "min_cosine": 0.99999,
            "rtol": 1e-05
          },
          "baseline_loss": {
            "atol": 1e-06,
            "rtol": 1e-06
          },
          "reference_gradient": {
            "atol": 1e-05,
            "min_cosine": 0.9999,
            "rtol": 0.0001
          },
          "reference_loss": {
            "atol": 1e-05,
            "rtol": 1e-05
          }
        }
      }
    ],
    "required": true,
    "status": "unresolved: the GS1 reference implementation and the tinker-cookbook revision are not yet pinned. No full run until both commit SHAs are recorded and checks pass."
  },
  "dataset": {
    "hf_path": "openai/gsm8k",
    "n_examples": 7473,
    "name": "gsm8k",
    "split": "train (config main)"
  },
  "decision_rule": {
    "alpha": 0.05,
    "criteria": [
      {
        "bounds": [
          0,
          1
        ],
        "id": "primary_noninferiority",
        "margin": -0.05,
        "metric": "accuracy",
        "weights": {
          "baseline": -1,
          "treatment": 1
        }
      }
    ],
    "estimand": "Difference in GSM8K test exact-match accuracy (GS1 minus GRPO) of Qwen3.5-2B LoRA after 40 steps at 64 rollouts/step, larger is better",
    "method": "paired_hoeffding",
    "seeds": [
      0
    ],
    "unit": "fixed_checkpoint_items",
    "version": 1
  },
  "eval": {
    "aggregation": null,
    "benchmark": "GSM8K test",
    "metric": "exact-match accuracy (greedy, final numeric answer)",
    "n_examples": 1319
  },
  "experiment_features": {
    "adaptive_selection": false,
    "checkpoint_restore": false,
    "custom_loss": true,
    "custom_optimizer": false,
    "generated_training_data": false,
    "heldout_loss": false,
    "learned_generator": false,
    "paired_comparison": true,
    "paired_training_arms": true,
    "warmup_adaptation": false
  },
  "feasibility": null,
  "feasibility_check": {
    "gpu_hours": null,
    "known_gpu_hours": 0,
    "phases": {},
    "problems": [
      "Measured feasibility not recorded"
    ],
    "status": "unknown",
    "wall_seconds": null
  },
  "feasibility_required": true,
  "generation_audit": {
    "answer_comparison": "numeric_exact",
    "max_eval_overlaps": 0,
    "max_label_errors": 0,
    "min_sample_size": 100,
    "required": false
  },
  "hypothesis": "At an equal budget of 64 rollouts per step for 40 steps, GS1 reaches GSM8K test exact-match accuracy no more than 5 points below GRPO with group size 8, from the same LoRA initialization.",
  "lora_rank": 32,
  "max_seq_len": 1024,
  "method": "Triage saw only the Slack summary of trajectory.ai's 'You Only Rollout Once'. No paper brief, code or hyperparameters were fetched, so the similarity function and temperature below are our assumption and must be checked against the paper before the full run. Both arms run in one custom tinker SDK script built on the tinker-cookbook RL loop (rl/train.py with the math_rl gsm8k env, same renderer and reward). The script has an advantage_mode switch. Only the advantage computation is custom; the server loss is the stock importance_sampling loss.\n\nBaseline arm (advantage_mode=grpo): each step samples 8 prompts x 8 rollouts. Advantage = reward minus the group mean, as in stock cookbook.\n\nTreatment arm (advantage_mode=gs1): each step samples 64 distinct prompts x 1 rollout.\n- The advantage for rollout i is r_i - b_i.\n- b_i = sum over j != i of w_ij * r_j, with w_ij = softmax over j != i of (cos(e_i, e_j) / tau).\n- e is a TF-IDF vector of the prompt text, and tau = 0.1. Both are assumptions pending the paper.\n- Before the GS1 arm runs, also log the offline B_eff of the uniform-LOO baseline and the similarity baseline on the pilot rollouts. This is a diagnostic only, not the endpoint.\n\nShared settings:\n- Model: Qwen/Qwen3.5-2B, LoRA rank 32, AdamW lr 4e-5, max_tokens 512, temperature 1.0.\n- Each arm: 40 steps, 1 seed, fresh LoRA from the same init seed.\n- Prompts: GSM8K train in a fixed shuffled order. GRPO consumes 320 prompts and GS1 consumes 2560.\n- Eval: greedy decoding on the full GSM8K test split (1319 items), exact match on the final number, from the step-40 checkpoint of each arm.\n\nThe correctness preflight must show that advantage_mode=grpo reproduces the stock cookbook loss and gradient on fixed rollouts. GS1 advantages are compared against a reference implementation pinned to a revision. That revision is currently unresolved, so the GS1 run is blocked until it is pinned.",
  "optimizer": {
    "lr": 4e-05,
    "name": "adam"
  },
  "primary_endpoint": "gsm8k_test_exact_match_accuracy_step40",
  "restoration_preflight": {
    "applicable": true,
    "fixed_batch_required": true,
    "groups": [
      {
        "arms": [
          {
            "name": "grpo_gs8_seed0",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "init_seed": 0,
              "learning_rate": 4e-05,
              "lora_rank": 32
            }
          },
          {
            "name": "gs1_seed0",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "init_seed": 0,
              "learning_rate": 4e-05,
              "lora_rank": 32
            }
          }
        ]
      }
    ],
    "reason": "Both arms train fresh LoRAs from the same base model and init seed. Initial identity is verified by forward logprobs on a fixed tokenized batch before the first update, along with the optimizer config observed after creation. Server identity beyond this stays unverified."
  },
  "steps": 40,
  "success_criterion": "GS1 minus GRPO paired accuracy on the 1319 GSM8K test items has a 95% two-sided Hoeffding lower bound above -0.05. The Hoeffding half-width at n=1319 is about 0.075, so this passes only if GS1's point estimate is at least ~2.5 points above GRPO. Otherwise the result is inconclusive, not a failure, unless the upper bound is below -0.05.",
  "title": "GS1 (group size 1, similarity-weighted LOO baseline) vs GRPO (group size 8) on GSM8K with Qwen3.5-2B",
  "training_type": "rl"
}

[
  {
    "arm": "shared_init",
    "step": 0,
    "uri": "tinker://66bc785a6dd2ae0f72c6707928dfb24b/weights/run_cbb327f6ab-init"
  },
  {
    "arm": "grpo_gs8_seed0",
    "step": 3,
    "uri": "tinker://57184424-e5cb-474b-9669-aad12c28d69d:train:1/weights/run_cbb327f6ab-grpo_gs8_seed0-pilot-final"
  },
  {
    "arm": "gs1_seed0",
    "step": 3,
    "uri": "tinker://57184424-e5cb-474b-9669-aad12c28d69d:train:2/weights/run_cbb327f6ab-gs1_seed0-pilot-final"
  }
]

{
  "grpo_gs8_seed0": {
    "pilot_mean_train_reward": 0.5953125,
    "pilot_seconds": 207.53177285194397
  },
  "gs1_seed0": {
    "pilot_mean_train_reward": 0.6130208333333335,
    "pilot_seconds": 203.69979238510132
  }
}

base_model_gsm8k_test_accuracy

0.7324

gs1_effective_contributors_median

57.71

gs1_mean_abs_baseline_minus_uniform_loo

0.02577

batch_mean_reward_pilot

0.613

b_eff_uniform_loo_pilot

14.08

b_eff_gs1_tfidf_pilot

14.11

• checkpoint · shared_init · step 0 · tinker://66bc785a6dd2ae0f72c6707928dfb24b/weights/run_cbb327f6ab-init
• checkpoint · grpo_gs8_seed0 · step 3 · tinker://57184424-e5cb-474b-9669-aad12c28d69d:train:1/weights/run_cbb327f6ab-grpo_gs8_seed0-pilot-final
• checkpoint · gs1_seed0 · step 3 · tinker://57184424-e5cb-474b-9669-aad12c28d69d:train:2/weights/run_cbb327f6ab-gs1_seed0-pilot-final
• script · gs1.py sha256 d096b3d671e6
• script · gs1_reference.py sha256 0abdddae2cfb
• script · common.py sha256 7efd0dcebf90
• script · preflight.py sha256 0c61ea42aedf
• script · analyze_pilot.py sha256 50044c487fca
• script · evaluate.py sha256 145609f3fe1b
• effective_config · effective-config.json
• code_revision · code-revision.txt
• metrics · restoration.json sha256 7533ed3c9d7c
• metrics · activation-calibration.json sha256 4e306093b60a
• metrics · correctness-preflight.json sha256 a04a04fa9998
• metrics · eval-base.json sha256 c7bdc66586e1
• metrics · pilot_results.json sha256 c0b2be0f440d
• predictions · predictions-base-model.jsonl sha256 08ba306f1dbc
• metrics · correctness-fixture-grpo_gs8_seed0.json sha256 f79a49f03bab
• metrics · correctness-fixture-gs1_seed0.json sha256 cfbcc17e8d8e
• metrics · evaluator.json sha256 1c614b787762

1. Nov 3, 21:33 UTC · failed · Restoration, correctness (GRPO vs stock cookbook; GS1 vs independent float64 reference) and a 3-step pilot of both arms ran on Spindle with Qwen3.5-2B; base model scores 0.7324 on GSM8K test. The pre…
2. Nov 3, 21:31 UTC · running · Restoration and correctness preflights passed; the 3-step GS1 activation pilot failed the gs1_effective_contributors_median gate (57.7 vs accepted 2-40), so full training will not start.
3. Nov 3, 21:25 UTC · running · Launch instructions downloaded, SHA256 verified (ea7286a5...), fully read. Cookbook pinned at 6daabe0838736557f0e802020b710073eb340287 (tinker 0.37.0). GS1 implementation and independent float64 refe…

correctness_preflight

{'required': True, 'paths': [{'id': 'gs1_advantage_rl_loss', 'fixed_input': {'sha256': 'a5f33a71f4a870d0758e9060568b92cd345b1b0702d573363982bb453bbb710a', 'description': 'baseline: step-0 pilot rollouts of grpo_gs8_seed0 (64 datums) sampled from shared init weights; one server forward (importance_sampling) on the never-updated init client for logprobs (sha256 5a3c4a874a1f1cc40ac52449a26c12c3d079d4b2e1e97e63593c0a3bebfb0fed, out/fixture_grpo_gs8_seed0.pkl); reference: step-0 pilot rollouts of gs1_seed0 (64 datums) sampled from shared init weights; one server forward (importance_sampling) on the never-updated init client for logprobs (sha256 b94b5a06547c019cb94c2c679970b1d01a7796aaf5767e2d9e12ca247d9f334a, out/fixture_gs1_seed0.pkl); optimizer state fresh from shared init'}, 'baseline_revision': 'UNRESOLVED: pin tinker-cookbook commit SHA (rl/train.py, importance_sampling loss) before running', 'reference_revision': "UNRESOLVED: pin trajectory.ai 'You Only Rollout Once' reference code commit SHA, or a reviewed independent float64 reference implementation at a pinned SHA", 'measured_baseline_revision': 'thinking-machines-lab/tinker-cookbook@6daabe0838736557f0e802020b710073eb340287 (compute_advantages + assemble_training_data, importance_sampling)', 'measured_reference_revision': 'independent float64 pure-Python reference code/gs1_reference.py sha256 0abdddae2cfbd20e575053c5d3727af414d3c07cba19b1cb8bc3356e93650f65', 'implementation_revision': 'content sha256 be66be41ae23d88b4a2cb73cfeea1756f8ee261950645877567c2b7ff1a181c2 (code/gs1.py, gs1_reference.py, preflight.py, common.py)', 'precision': 'float64 local loss/grad over server float32 logprobs; advantages sent as float32', 'normalization': 'importance_sampling sum over tokens per datum (Tinker loss:sum), no per-batch normalization', 'errors': {'baseline_loss': {'max_abs_error': 3.0567533258363255e-05, 'reference_max_abs': 492.42219406518177, 'relative_l2_error': 6.937014880483375e-08, 'cosine_similarity': 0.9999999999999986}, 'baseline_gradient': {'max_abs_error': 7.15193320122154e-08, 'reference_max_abs': 1.135085226398739, 'relative_l2_error': 6.609284206365097e-08, 'cosine_similarity': 0.9999999999999982}, 'reference_loss': {'max_abs_error': 1.4858435463338536e-05, 'reference_max_abs': 356.5939284235209, 'relative_l2_error': 2.7288668728921185e-08, 'cosine_similarity': 0.9999999999999998}, 'reference_gradient': {'max_abs_error': 3.2636295665966486e-08, 'reference_max_abs': 0.8579570351141478, 'relative_l2_error': 2.6648511609619557e-08, 'cosine_similarity': 1.0}}, 'lr_scaling': []}]}

correctness_validation

{'status': 'passed', 'required': True, 'checks': [{'path': 'gs1_advantage_rl_loss', 'check': 'baseline_loss', 'status': 'passed', 'tolerance': {'atol': 1e-06, 'rtol': 1e-06}, 'measured': {'max_abs_error': 3.0567533258363255e-05, 'reference_max_abs': 492.42219406518177, 'relative_l2_error': 6.937014880483375e-08, 'cosine_similarity': 0.9999999999999986}}, {'path': 'gs1_advantage_rl_loss', 'check': 'baseline_gradient', 'status': 'passed', 'tolerance': {'atol': 1e-06, 'rtol': 1e-05, 'min_cosine': 0.99999}, 'measured': {'max_abs_error': 7.15193320122154e-08, 'reference_max_abs': 1.135085226398739, 'relative_l2_error': 6.609284206365097e-08, 'cosine_similarity': 0.9999999999999982}}, {'path': 'gs1_advantage_rl_loss', 'check': 'reference_loss', 'status': 'passed', 'tolerance': {'atol': 1e-05, 'rtol': 1e-05}, 'measured': {'max_abs_error': 1.4858435463338536e-05, 'reference_max_abs': 356.5939284235209, 'relative_l2_error': 2.7288668728921185e-08, 'cosine_similarity': 0.9999999999999998}}, {'path': 'gs1_advantage_rl_loss', 'check': 'reference_gradient', 'status': 'passed', 'tolerance': {'atol': 1e-05, 'rtol': 0.0001, 'min_cosine': 0.9999}, 'measured': {'max_abs_error': 3.2636295665966486e-08, 'reference_max_abs': 0.8579570351141478, 'relative_l2_error': 2.6648511609619557e-08, 'cosine_similarity': 1.0}}], 'reason': 'Reported baseline and reference parity checks meet accepted tolerances.', 'evidence_source': 'executor report; remote execution is not attested', 'applicability': 'required', 'reported_verdict': 'inconclusive'}

decision_validation

{'status': 'uncertain', 'issues': ['Decision evidence was not recorded.'], 'criteria': [], 'outcome_reason': 'insufficient_evidence', 'unit': 'fixed_checkpoint_items', 'n_seeds': 1, 'alpha': 0.05, 'method': 'paired_hoeffding', 'applicability': {'check': 'decision_rule', 'status': 'required', 'enforce': True, 'reasons': ['experiment_features: paired_comparison is true'], 'features': {'paired_comparison': True, 'adaptive_selection': False}}}

evaluation

{'version': 1, 'evaluators': {'3bdc9adc3aa8c8f2f9d9e3190001ca7f06d2cf27a204f3b71d02d6983e524423': {'code_sha256': 'c31cad0637784130acfed8a3a541939957106a0d6a9923c3982a91296087ac58', 'prompt_template': {'renderer': 'qwen3_5_disable_thinking (tinker-cookbook@6daabe08)', 'tokenizer': 'Qwen/Qwen3.5-2B', 'messages': "MathEnv.standard_fewshot_prefix() (strawberry r-count 1-shot) + [user: question + ' Provide a numerical answer without units, written inside \\\\boxed{}.']", 'generation_prompt': 'renderer.build_generation_prompt(messages)'}, 'decoding': {'temperature': 0, 'max_tokens': 512, 'num_samples': 1, 'stop': 'renderer.get_stop_sequences() for qwen3_5_disable_thinking'}, 'parser': "tinker_cookbook.recipes.math_rl.math_grading.extract_boxed (last \\boxed{}) -> strip ',', '$', spaces, trailing '.', \\text{} wrapper -> fractions.Fraction equality vs extract_gsm8k_final_answer(gold); missing/unparsable -> 0 (code/evaluate.py score/to_num)", 'items': {'ids': ['gsm8k-test-0000', 'gsm8k-test-0001', 'gsm8k-test-0002', 'gsm8k-test-0003', 'gsm8k-test-0004', 'gsm8k-test-0005', 'gsm8k-test-0006', 'gsm8k-test-0007', 'gsm8k-test-0008', 'gsm8k-test-0009', 'gsm8k-test-0010', 'gsm8k-test-0011', 'gsm8k-test-0012', 'gsm8k-test-0013', 'gsm8k-test-0014', 'gsm8k-test-0015', 'gsm8k-test-0016', 'gsm8k-test-0017', 'gsm8k-test-0018', 'gsm8k-test-0019', 'gsm8k-test-0020', 'gsm8k-test-0021', 'gsm8k-test-0022', 'gsm8k-test-0023', 'gsm8k-test-0024', 'gsm8k-test-0025', 'gsm8k-test-0026', 'gsm8k-test-0027', 'gsm8k-test-0028', 'gsm8k-test-0029', 'gsm8k-test-0030', 'gsm8k-test-0031', 'gsm8k-test-0032', 'gsm8k-test-0033', 'gsm8k-test-0034', 'gsm8k-test-0035', 'gsm8k-test-0036', 'gsm8k-test-0037', 'gsm8k-test-0038', 'gsm8k-test-0039', 'gsm8k-test-0040', 'gsm8k-test-0041', 'gsm8k-test-0042', 'gsm8k-test-0043', 'gsm8k-test-0044', 'gsm8k-test-0045', 'gsm8k-test-0046', 'gsm8k-test-0047', 'gsm8k-test-0048', 'gsm8k-test-0049', 'gsm8k-test-0050', 'gsm8k-test-0051', 'gsm8k-test-0052', 'gsm8k-test-0053', 'gsm8k-test-0054', 'gsm8k-test-0055', 'gsm8k-test-0056', 'gsm8k-test-0057', 'gsm8k-test-0058', 'gsm8k-test-0059', 'gsm8k-test-0060', 'gsm8k-test-0061', 'gsm8k-test-0062', 'gsm8k-test-0063', 'gsm8k-test-0064', 'gsm8k-test-0065', 'gsm8k-test-0066', 'gsm8k-test-0067', 'gsm8k-test-0068', 'gsm8k-test-0069', 'gsm8k-test-0070', 'gsm8k-test-0071', 'gsm8k-test-0072', 'gsm8k-test-0073', 'gsm8k-test-0074', 'gsm8k-test-0075', 'gsm8k-test-0076', 'gsm8k-test-0077', 'gsm8k-test-0078', 'gsm8k-test-0079', 'gsm8k-test-0080', 'gsm8k-test-0081', 'gsm8k-test-0082', 'gsm8k-test-0083', 'gsm8k-test-0084', 'gsm8k-test-0085', 'gsm8k-test-0086', 'gsm8k-test-0087', 'gsm8k-test-0088', 'gsm8k-test-0089', 'gsm8k-test-0090', 'gsm8k-test-0091', 'gsm8k-test-0092', 'gsm8k-test-0093', 'gsm8k-test-0094', 'gsm8k-test-0095', 'gsm8k-test-0096', 'gsm8k-test-0097', 'gsm8k-test-0098', 'gsm8k-test-0099', 'gsm8k-test-0100', 'gsm8k-test-0101', 'gsm8k-test-0102', 'gsm8k-test-0103', 'gsm8k-test-0104', 'gsm8k-test-0105', 'gsm8k-test-0106', 'gsm8k-test-0107', 'gsm8k-test-0108', 'gsm8k-test-0109', 'gsm8k-test-0110', 'gsm8k-test-0111', 'gsm8k-test-0112', 'gsm8k-test-0113', 'gsm8k-test-0114', 'gsm8k-test-0115', 'gsm8k-test-0116', 'gsm8k-test-0117', 'gsm8k-test-0118', 'gsm8k-test-0119', 'gsm8k-test-0120', 'gsm8k-test-0121', 'gsm8k-test-0122', 'gsm8k-test-0123', 'gsm8k-test-0124', 'gsm8k-test-0125', 'gsm8k-test-0126', 'gsm8k-test-0127', 'gsm8k-test-0128', 'gsm8k-test-0129', 'gsm8k-test-0130', 'gsm8k-test-0131', 'gsm8k-test-0132', 'gsm8k-test-0133', 'gsm8k-test-0134', 'gsm8k-test-0135', 'gsm8k-test-0136', 'gsm8k-test-0137', 'gsm8k-test-0138', 'gsm8k-test-0139', 'gsm8k-test-0140', 'gsm8k-test-0141', 'gsm8k-test-0142', 'gsm8k-test-0143', 'gsm8k-test-0144', 'gsm8k-test-0145', 'gsm8k-test-0146', 'gsm8k-test-0147', 'gsm8k-test-0148', 'gsm8k-test-0149', 'gsm8k-test-0150', 'gsm8k-test-0151', 'gsm8k-test-0152', 'gsm8k-test-0153', 'gsm8k-test-0154', 'gsm8k-test-0155', 'gsm8k-test-0156', 'gsm8k-test-0157', 'gsm8k-test-0158', 'gsm8k-test-0159', 'gsm8k-test-0160', 'gsm8k-test-0161', 'gsm8k-test-0162', 'gsm8k-test-0163', 'gsm8k-test-0164', 'gsm8k-test-0165', 'gsm8k-test-0166', 'gsm8k-test-0167', 'gsm8k-test-0168', 'gsm8k-test-0169', 'gsm8k-test-0170', 'gsm8k-test-0171', 'gsm8k-test-0172', 'gsm8k-test-0173', 'gsm8k-test-0174', 'gsm8k-test-0175', 'gsm8k-test-0176', 'gsm8k-test-0177', 'gsm8k-test-0178', 'gsm8k-test-0179', 'gsm8k-test-0180', 'gsm8k-test-0181', 'gsm8k-test-0182', 'gsm8k-test-0183', 'gsm8k-test-0184', 'gsm8k-test-0185', 'gsm8k-test-0186', 'gsm8k-test-0187', 'gsm8k-test-0188', 'gsm8k-test-0189', 'gsm8k-test-0190', 'gsm8k-test-0191', 'gsm8k-test-0192', 'gsm8k-test-0193', 'gsm8k-test-0194', 'gsm8k-test-0195', 'gsm8k-test-0196', 'gsm8k-test-0197', 'gsm8k-test-0198', 'gsm8k-test-0199', 'gsm8k-test-0200', 'gsm8k-test-0201', 'gsm8k-test-0202', 'gsm8k-test-0203', 'gsm8k-test-0204', 'gsm8k-test-0205', 'gsm8k-test-0206', 'gsm8k-test-0207', 'gsm8k-test-0208', 'gsm8k-test-0209', 'gsm8k-test-0210', 'gsm8k-test-0211', 'gsm8k-test-0212', 'gsm8k-test-0213', 'gsm8k-test-0214', 'gsm8k-test-0215', 'gsm8k-test-0216', 'gsm8k-test-0217', 'gsm8k-test-0218', 'gsm8k-test-0219', 'gsm8k-test-0220', 'gsm8k-test-0221', 'gsm8k-test-0222', 'gsm8k-test-0223', 'gsm8k-test-0224', 'gsm8k-test-0225', 'gsm8k-test-0226', 'gsm8k-test-0227', 'gsm8k-test-0228', 'gsm8k-test-0229', 'gsm8k-test-0230', 'gsm8k-test-0231', 'gsm8k-test-0232', 'gsm8k-test-0233', 'gsm8k-test-0234', 'gsm8k-test-0235', 'gsm8k-test-0236', 'gsm8k-test-0237', 'gsm8k-test-0238', 'gsm8k-test-0239', 'gsm8k-test-0240', 'gsm8k-test-0241', 'gsm8k-test-0242', 'gsm8k-test-0243', 'gsm8k-test-0244', 'gsm8k-test-0245', 'gsm8k-test-0246', 'gsm8k-test-0247', 'gsm8k-test-0248', 'gsm8k-test-0249', 'gsm8k-test-0250', 'gsm8k-test-0251', 'gsm8k-test-0252', 'gsm8k-test-0253', 'gsm8k-test-0254', 'gsm8k-test-0255', 'gsm8k-test-0256', 'gsm8k-test-0257', 'gsm8k-test-0258', 'gsm8k-test-0259', 'gsm8k-test-0260', 'gsm8k-test-0261', 'gsm8k-test-0262', 'gsm8k-test-0263', 'gsm8k-test-0264', 'gsm8k-test-0265', 'gsm8k-test-0266', 'gsm8k-test-0267', 'gsm8k-test-0268', 'gsm8k-test-0269', 'gsm8k-test-0270', 'gsm8k-test-0271', 'gsm8k-test-0272', 'gsm8k-test-0273', 'gsm8k-test-0274', 'gsm8k-test-0275', 'gsm8k-test-0276', 'gsm8k-test-0277', 'gsm8k-test-0278', 'gsm8k-test-0279', 'gsm8k-test-0280', 'gsm8k-test-0281', 'gsm8k-test-0282', 'gsm8k-test-0283', 'gsm8k-test-0284', 'gsm8k-test-0285', 'gsm8k-test-0286', 'gsm8k-test-0287', 'gsm8k-test-0288', 'gsm8k-test-0289', 'gsm8k-test-0290', 'gsm8k-test-0291', 'gsm8k-test-0292', 'gsm8k-test-0293', 'gsm8k-test-0294', 'gsm8k-test-0295', 'gsm8k-test-0296', 'gsm8k-test-0297', 'gsm8k-test-0298', 'gsm8k-test-0299', 'gsm8k-test-0300', 'gsm8k-test-0301', 'gsm8k-test-0302', 'gsm8k-test-0303', 'gsm8k-test-0304', 'gsm8k-test-0305', 'gsm8k-test-0306', 'gsm8k-test-0307', 'gsm8k-test-0308', 'gsm8k-test-0309', 'gsm8k-test-0310', 'gsm8k-test-0311', 'gsm8k-test-0312', 'gsm8k-test-0313', 'gsm8k-test-0314', 'gsm8k-test-0315', 'gsm8k-test-0316', 'gsm8k-test-0317', 'gsm8k-test-0318', 'gsm8k-test-0319', 'gsm8k-test-0320', 'gsm8k-test-0321', 'gsm8k-test-0322', 'gsm8k-test-0323', 'gsm8k-test-0324', 'gsm8k-test-0325', 'gsm8k-test-0326', 'gsm8k-test-0327', 'gsm8k-test-0328', 'gsm8k-test-0329', 'gsm8k-test-0330', 'gsm8k-test-0331', 'gsm8k-test-0332', 'gsm8k-test-0333', 'gsm8k-test-0334', 'gsm8k-test-0335', 'gsm8k-test-0336', 'gsm8k-test-0337', 'gsm8k-test-0338', 'gsm8k-test-0339', 'gsm8k-test-0340', 'gsm8k-test-0341', 'gsm8k-test-0342', 'gsm8k-test-0343', 'gsm8k-test-0344', 'gsm8k-test-0345', 'gsm8k-test-0346', 'gsm8k-test-0347', 'gsm8k-test-0348', 'gsm8k-test-0349', 'gsm8k-test-0350', 'gsm8k-test-0351', 'gsm8k-test-0352', 'gsm8k-test-0353', 'gsm8k-test-0354', 'gsm8k-test-0355', 'gsm8k-test-0356', 'gsm8k-test-0357', 'gsm8k-test-0358', 'gsm8k-test-0359', 'gsm8k-test-0360', 'gsm8k-test-0361', 'gsm8k-test-0362', 'gsm8k-test-0363', 'gsm8k-test-0364', 'gsm8k-test-0365', 'gsm8k-test-0366', 'gsm8k-test-0367', 'gsm8k-test-0368', 'gsm8k-test-0369', 'gsm8k-test-0370', 'gsm8k-test-0371', 'gsm8k-test-0372', 'gsm8k-test-0373', 'gsm8k-test-0374', 'gsm8k-test-0375', 'gsm8k-test-0376', 'gsm8k-test-0377', 'gsm8k-test-0378', 'gsm8k-test-0379', 'gsm8k-test-0380', 'gsm8k-test-0381', 'gsm8k-test-0382', 'gsm8k-test-0383', 'gsm8k-test-0384', 'gsm8k-test-0385', 'gsm8k-test-0386', 'gsm8k-test-0387', 'gsm8k-test-0388', 'gsm8k-test-0389', 'gsm8k-test-0390', 'gsm8k-test-0391', 'gsm8k-test-0392', 'gsm8k-test-0393', 'gsm8k-test-0394', 'gsm8k-test-0395', 'gsm8k-test-0396', 'gsm8k-test-0397', 'gsm8k-test-0398', 'gsm8k-test-0399', 'gsm8k-test-0400', 'gsm8k-test-0401', 'gsm8k-test-0402', 'gsm8k-test-0403', 'gsm8k-test-0404', 'gsm8k-test-0405', 'gsm8k-test-0406', 'gsm8k-test-0407', 'gsm8k-test-0408', 'gsm8k-test-0409', 'gsm8k-test-0410', 'gsm8k-test-0411', 'gsm8k-test-0412', 'gsm8k-test-0413', 'gsm8k-test-0414', 'gsm8k-test-0415', 'gsm8k-test-0416', 'gsm8k-test-0417', 'gsm8k-test-0418', 'gsm8k-test-0419', 'gsm8k-test-0420', 'gsm8k-test-0421', 'gsm8k-test-0422', 'gsm8k-test-0423', 'gsm8k-test-0424', 'gsm8k-test-0425', 'gsm8k-test-0426', 'gsm8k-test-0427', 'gsm8k-test-0428', 'gsm8k-test-0429', 'gsm8k-test-0430', 'gsm8k-test-0431', 'gsm8k-test-0432', 'gsm8k-test-0433', 'gsm8k-test-0434', 'gsm8k-test-0435', 'gsm8k-test-0436', 'gsm8k-test-0437', 'gsm8k-test-0438', 'gsm8k-test-0439', 'gsm8k-test-0440', 'gsm8k-test-0441', 'gsm8k-test-0442', 'gsm8k-test-0443', 'gsm8k-test-0444', 'gsm8k-test-0445', 'gsm8k-test-0446', 'gsm8k-test-0447', 'gsm8k-test-0448', 'gsm8k-test-0449', 'gsm8k-test-0450', 'gsm8k-test-0451', 'gsm8k-test-0452', 'gsm8k-test-0453', 'gsm8k-test-0454', 'gsm8k-test-0455', 'gsm8k-test-0456', 'gsm8k-test-0457', 'gsm8k-test-0458', 'gsm8k-test-0459', 'gsm8k-test-0460', 'gsm8k-test-0461', 'gsm8k-test-0462', 'gsm8k-test-0463', 'gsm8k-test-0464', 'gsm8k-test-0465', 'gsm8k-test-0466', 'gsm8k-test-0467', 'gsm8k-test-0468', 'gsm8k-test-0469', 'gsm8k-test-0470', 'gsm8k-test-0471', 'gsm8k-test-0472', 'gsm8k-test-0473', 'gsm8k-test-0474', 'gsm8k-test-0475', 'gsm8k-test-0476', 'gsm8k-test-0477', 'gsm8k-test-0478', 'gsm8k-test-0479', 'gsm8k-test-0480', 'gsm8k-test-0481', 'gsm8k-test-0482', 'gsm8k-test-0483', 'gsm8k-test-0484', 'gsm8k-test-0485', 'gsm8k-test-0486', 'gsm8k-test-0487', 'gsm8k-test-0488', 'gsm8k-test-0489', 'gsm8k-test-0490', 'gsm8k-test-0491', 'gsm8k-test-0492', 'gsm8k-test-0493', 'gsm8k-test-0494', 'gsm8k-test-0495', 'gsm8k-test-0496', 'gsm8k-test-0497', 'gsm8k-test-0498', 'gsm8k-test-0499', 'gsm8k-test-0500', 'gsm8k-test-0501', 'gsm8k-test-0502', 'gsm8k-test-0503', 'gsm8k-test-0504', 'gsm8k-test-0505', 'gsm8k-test-0506', 'gsm8k-test-0507', 'gsm8k-test-0508', 'gsm8k-test-0509', 'gsm8k-test-0510', 'gsm8k-test-0511', 'gsm8k-test-0512', 'gsm8k-test-0513', 'gsm8k-test-0514', 'gsm8k-test-0515', 'gsm8k-test-0516', 'gsm8k-test-0517', 'gsm8k-test-0518', 'gsm8k-test-0519', 'gsm8k-test-0520', 'gsm8k-test-0521', 'gsm8k-test-0522', 'gsm8k-test-0523', 'gsm8k-test-0524', 'gsm8k-test-0525', 'gsm8k-test-0526', 'gsm8k-test-0527', 'gsm8k-test-0528', 'gsm8k-test-0529', 'gsm8k-test-0530', 'gsm8k-test-0531', 'gsm8k-test-0532', 'gsm8k-test-0533', 'gsm8k-test-0534', 'gsm8k-test-0535', 'gsm8k-test-0536', 'gsm8k-test-0537', 'gsm8k-test-0538', 'gsm8k-test-0539', 'gsm8k-test-0540', 'gsm8k-test-0541', 'gsm8k-test-0542', 'gsm8k-test-0543', 'gsm8k-test-0544', 'gsm8k-test-0545', 'gsm8k-test-0546', 'gsm8k-test-0547', 'gsm8k-test-0548', 'gsm8k-test-0549', 'gsm8k-test-0550', 'gsm8k-test-0551', 'gsm8k-test-0552', 'gsm8k-test-0553', 'gsm8k-test-0554', 'gsm8k-test-0555', 'gsm8k-test-0556', 'gsm8k-test-0557', 'gsm8k-test-0558', 'gsm8k-test-0559', 'gsm8k-test-0560', 'gsm8k-test-0561', 'gsm8k-test-0562', 'gsm8k-test-0563', 'gsm8k-test-0564', 'gsm8k-test-0565', 'gsm8k-test-0566', 'gsm8k-test-0567', 'gsm8k-test-0568', 'gsm8k-test-0569', 'gsm8k-test-0570', 'gsm8k-test-0571', 'gsm8k-test-0572', 'gsm8k-test-0573', 'gsm8k-test-0574', 'gsm8k-test-0575', 'gsm8k-test-0576', 'gsm8k-test-0577', 'gsm8k-test-0578', 'gsm8k-test-0579', 'gsm8k-test-0580', 'gsm8k-test-0581', 'gsm8k-test-0582', 'gsm8k-test-0583', 'gsm8k-test-0584', 'gsm8k-test-0585', 'gsm8k-test-0586', 'gsm8k-test-0587', 'gsm8k-test-0588', 'gsm8k-test-0589', 'gsm8k-test-0590', 'gsm8k-test-0591', 'gsm8k-test-0592', 'gsm8k-test-0593', 'gsm8k-test-0594', 'gsm8k-test-0595', 'gsm8k-test-0596', 'gsm8k-test-0597', 'gsm8k-test-0598', 'gsm8k-test-0599', 'gsm8k-test-0600', 'gsm8k-test-0601', 'gsm8k-test-0602', 'gsm8k-test-0603', 'gsm8k-test-0604', 'gsm8k-test-0605', 'gsm8k-test-0606', 'gsm8k-test-0607', 'gsm8k-test-0608', 'gsm8k-test-0609', 'gsm8k-test-0610', 'gsm8k-test-0611', 'gsm8k-test-0612', 'gsm8k-test-0613', 'gsm8k-test-0614', 'gsm8k-test-0615', 'gsm8k-test-0616', 'gsm8k-test-0617', 'gsm8k-test-0618', 'gsm8k-test-0619', 'gsm8k-test-0620', 'gsm8k-test-0621', 'gsm8k-test-0622', 'gsm8k-test-0623', 'gsm8k-test-0624', 'gsm8k-test-0625', 'gsm8k-test-0626', 'gsm8k-test-0627', 'gsm8k-test-0628', 'gsm8k-test-0629', 'gsm8k-test-0630', 'gsm8k-test-0631', 'gsm8k-test-0632', 'gsm8k-test-0633', 'gsm8k-test-0634', 'gsm8k-test-0635', 'gsm8k-test-0636', 'gsm8k-test-0637', 'gsm8k-test-0638', 'gsm8k-test-0639', 'gsm8k-test-0640', 'gsm8k-test-0641', 'gsm8k-test-0642', 'gsm8k-test-0643', 'gsm8k-test-0644', 'gsm8k-test-0645', 'gsm8k-test-0646', 'gsm8k-test-0647', 'gsm8k-test-0648', 'gsm8k-test-0649', 'gsm8k-test-0650', 'gsm8k-test-0651', 'gsm8k-test-0652', 'gsm8k-test-0653', 'gsm8k-test-0654', 'gsm8k-test-0655', 'gsm8k-test-0656', 'gsm8k-test-0657', 'gsm8k-test-0658', 'gsm8k-test-0659', 'gsm8k-test-0660', 'gsm8k-test-0661', 'gsm8k-test-0662', 'gsm8k-test-0663', 'gsm8k-test-0664', 'gsm8k-test-0665', 'gsm8k-test-0666', 'gsm8k-test-0667', 'gsm8k-test-0668', 'gsm8k-test-0669', 'gsm8k-test-0670', 'gsm8k-test-0671', 'gsm8k-test-0672', 'gsm8k-test-0673', 'gsm8k-test-0674', 'gsm8k-test-0675', 'gsm8k-test-0676', 'gsm8k-test-0677', 'gsm8k-test-0678', 'gsm8k-test-0679', 'gsm8k-test-0680', 'gsm8k-test-0681', 'gsm8k-test-0682', 'gsm8k-test-0683', 'gsm8k-test-0684', 'gsm8k-test-0685', 'gsm8k-test-0686', 'gsm8k-test-0687', 'gsm8k-test-0688', 'gsm8k-test-0689', 'gsm8k-test-0690', 'gsm8k-test-0691', 'gsm8k-test-0692', 'gsm8k-test-0693', 'gsm8k-test-0694', 'gsm8k-test-0695', 'gsm8k-test-0696', 'gsm8k-test-0697', 'gsm8k-test-0698', 'gsm8k-test-0699', 'gsm8k-test-0700', 'gsm8k-test-0701', 'gsm8k-test-0702', 'gsm8k-test-0703', 'gsm8k-test-0704', 'gsm8k-test-0705', 'gsm8k-test-0706', 'gsm8k-test-0707', 'gsm8k-test-0708', 'gsm8k-test-0709', 'gsm8k-test-0710', 'gsm8k-test-0711', 'gsm8k-test-0712', 'gsm8k-test-0713', 'gsm8k-test-0714', 'gsm8k-test-0715', 'gsm8k-test-0716', 'gsm8k-test-0717', 'gsm8k-test-0718', 'gsm8k-test-0719', 'gsm8k-test-0720', 'gsm8k-test-0721', 'gsm8k-test-0722', 'gsm8k-test-0723', 'gsm8k-test-0724', 'gsm8k-test-0725', 'gsm8k-test-0726', 'gsm8k-test-0727', 'gsm8k-test-0728', 'gsm8k-test-0729', 'gsm8k-test-0730', 'gsm8k-test-0731', 'gsm8k-test-0732', 'gsm8k-test-0733', 'gsm8k-test-0734', 'gsm8k-test-0735', 'gsm8k-test-0736', 'gsm8k-test-0737', 'gsm8k-test-0738', 'gsm8k-test-0739', 'gsm8k-test-0740', 'gsm8k-test-0741', 'gsm8k-test-0742', 'gsm8k-test-0743', 'gsm8k-test-0744', 'gsm8k-test-0745', 'gsm8k-test-0746', 'gsm8k-test-0747', 'gsm8k-test-0748', 'gsm8k-test-0749', 'gsm8k-test-0750', 'gsm8k-test-0751', 'gsm8k-test-0752', 'gsm8k-test-0753', 'gsm8k-test-0754', 'gsm8k-test-0755', 'gsm8k-test-0756', 'gsm8k-test-0757', 'gsm8k-test-0758', 'gsm8k-test-0759', 'gsm8k-test-0760', 'gsm8k-test-0761', 'gsm8k-test-0762', 'gsm8k-test-0763', 'gsm8k-test-0764', 'gsm8k-test-0765', 'gsm8k-test-0766', 'gsm8k-test-0767', 'gsm8k-test-0768', 'gsm8k-test-0769', 'gsm8k-test-0770', 'gsm8k-test-0771', 'gsm8k-test-0772', 'gsm8k-test-0773', 'gsm8k-test-0774', 'gsm8k-test-0775', 'gsm8k-test-0776', 'gsm8k-test-0777', 'gsm8k-test-0778', 'gsm8k-test-0779', 'gsm8k-test-0780', 'gsm8k-test-0781', 'gsm8k-test-0782', 'gsm8k-test-0783', 'gsm8k-test-0784', 'gsm8k-test-0785', 'gsm8k-test-0786', 'gsm8k-test-0787', 'gsm8k-test-0788', 'gsm8k-test-0789', 'gsm8k-test-0790', 'gsm8k-test-0791', 'gsm8k-test-0792', 'gsm8k-test-0793', 'gsm8k-test-0794', 'gsm8k-test-0795', 'gsm8k-test-0796', 'gsm8k-test-0797', 'gsm8k-test-0798', 'gsm8k-test-0799', 'gsm8k-test-0800', 'gsm8k-test-0801', 'gsm8k-test-0802', 'gsm8k-test-0803', 'gsm8k-test-0804', 'gsm8k-test-0805', 'gsm8k-test-0806', 'gsm8k-test-0807', 'gsm8k-test-0808', 'gsm8k-test-0809', 'gsm8k-test-0810', 'gsm8k-test-0811', 'gsm8k-test-0812', 'gsm8k-test-0813', 'gsm8k-test-0814', 'gsm8k-test-0815', 'gsm8k-test-0816', 'gsm8k-test-0817', 'gsm8k-test-0818', 'gsm8k-test-0819', 'gsm8k-test-0820', 'gsm8k-test-0821', 'gsm8k-test-0822', 'gsm8k-test-0823', 'gsm8k-test-0824', 'gsm8k-test-0825', 'gsm8k-test-0826', 'gsm8k-test-0827', 'gsm8k-test-0828', 'gsm8k-test-0829', 'gsm8k-test-0830', 'gsm8k-test-0831', 'gsm8k-test-0832', 'gsm8k-test-0833', 'gsm8k-test-0834', 'gsm8k-test-0835', 'gsm8k-test-0836', 'gsm8k-test-0837', 'gsm8k-test-0838', 'gsm8k-test-0839', 'gsm8k-test-0840', 'gsm8k-test-0841', 'gsm8k-test-0842', 'gsm8k-test-0843', 'gsm8k-test-0844', 'gsm8k-test-0845', 'gsm8k-test-0846', 'gsm8k-test-0847', 'gsm8k-test-0848', 'gsm8k-test-0849', 'gsm8k-test-0850', 'gsm8k-test-0851', 'gsm8k-test-0852', 'gsm8k-test-0853', 'gsm8k-test-0854', 'gsm8k-test-0855', 'gsm8k-test-0856', 'gsm8k-test-0857', 'gsm8k-test-0858', 'gsm8k-test-0859', 'gsm8k-test-0860', 'gsm8k-test-0861', 'gsm8k-test-0862', 'gsm8k-test-0863', 'gsm8k-test-0864', 'gsm8k-test-0865', 'gsm8k-test-0866', 'gsm8k-test-0867', 'gsm8k-test-0868', 'gsm8k-test-0869', 'gsm8k-test-0870', 'gsm8k-test-0871', 'gsm8k-test-0872', 'gsm8k-test-0873', 'gsm8k-test-0874', 'gsm8k-test-0875', 'gsm8k-test-0876', 'gsm8k-test-0877', 'gsm8k-test-0878', 'gsm8k-test-0879', 'gsm8k-test-0880', 'gsm8k-test-0881', 'gsm8k-test-0882', 'gsm8k-test-0883', 'gsm8k-test-0884', 'gsm8k-test-0885', 'gsm8k-test-0886', 'gsm8k-test-0887', 'gsm8k-test-0888', 'gsm8k-test-0889', 'gsm8k-test-0890', 'gsm8k-test-0891', 'gsm8k-test-0892', 'gsm8k-test-0893', 'gsm8k-test-0894', 'gsm8k-test-0895', 'gsm8k-test-0896', 'gsm8k-test-0897', 'gsm8k-test-0898', 'gsm8k-test-0899', 'gsm8k-test-0900', 'gsm8k-test-0901', 'gsm8k-test-0902', 'gsm8k-test-0903', 'gsm8k-test-0904', 'gsm8k-test-0905', 'gsm8k-test-0906', 'gsm8k-test-0907', 'gsm8k-test-0908', 'gsm8k-test-0909', 'gsm8k-test-0910', 'gsm8k-test-0911', 'gsm8k-test-0912', 'gsm8k-test-0913', 'gsm8k-test-0914', 'gsm8k-test-0915', 'gsm8k-test-0916', 'gsm8k-test-0917', 'gsm8k-test-0918', 'gsm8k-test-0919', 'gsm8k-test-0920', 'gsm8k-test-0921', 'gsm8k-test-0922', 'gsm8k-test-0923', 'gsm8k-test-0924', 'gsm8k-test-0925', 'gsm8k-test-0926', 'gsm8k-test-0927', 'gsm8k-test-0928', 'gsm8k-test-0929', 'gsm8k-test-0930', 'gsm8k-test-0931', 'gsm8k-test-0932', 'gsm8k-test-0933', 'gsm8k-test-0934', 'gsm8k-test-0935', 'gsm8k-test-0936', 'gsm8k-test-0937', 'gsm8k-test-0938', 'gsm8k-test-0939', 'gsm8k-test-0940', 'gsm8k-test-0941', 'gsm8k-test-0942', 'gsm8k-test-0943', 'gsm8k-test-0944', 'gsm8k-test-0945', 'gsm8k-test-0946', 'gsm8k-test-0947', 'gsm8k-test-0948', 'gsm8k-test-0949', 'gsm8k-test-0950', 'gsm8k-test-0951', 'gsm8k-test-0952', 'gsm8k-test-0953', 'gsm8k-test-0954', 'gsm8k-test-0955', 'gsm8k-test-0956', 'gsm8k-test-0957', 'gsm8k-test-0958', 'gsm8k-test-0959', 'gsm8k-test-0960', 'gsm8k-test-0961', 'gsm8k-test-0962', 'gsm8k-test-0963', 'gsm8k-test-0964', 'gsm8k-test-0965', 'gsm8k-test-0966', 'gsm8k-test-0967', 'gsm8k-test-0968', 'gsm8k-test-0969', 'gsm8k-test-0970', 'gsm8k-test-0971', 'gsm8k-test-0972', 'gsm8k-test-0973', 'gsm8k-test-0974', 'gsm8k-test-0975', 'gsm8k-test-0976', 'gsm8k-test-0977', 'gsm8k-test-0978', 'gsm8k-test-0979', 'gsm8k-test-0980', 'gsm8k-test-0981', 'gsm8k-test-0982', 'gsm8k-test-0983', 'gsm8k-test-0984', 'gsm8k-test-0985', 'gsm8k-test-0986', 'gsm8k-test-0987', 'gsm8k-test-0988', 'gsm8k-test-0989', 'gsm8k-test-0990', 'gsm8k-test-0991', 'gsm8k-test-0992', 'gsm8k-test-0993', 'gsm8k-test-0994', 'gsm8k-test-0995', 'gsm8k-test-0996', 'gsm8k-test-0997', 'gsm8k-test-0998', 'gsm8k-test-0999', 'gsm8k-test-1000', 'gsm8k-test-1001', 'gsm8k-test-1002', 'gsm8k-test-1003', 'gsm8k-test-1004', 'gsm8k-test-1005', 'gsm8k-test-1006', 'gsm8k-test-1007', 'gsm8k-test-1008', 'gsm8k-test-1009', 'gsm8k-test-1010', 'gsm8k-test-1011', 'gsm8k-test-1012', 'gsm8k-test-1013', 'gsm8k-test-1014', 'gsm8k-test-1015', 'gsm8k-test-1016', 'gsm8k-test-1017', 'gsm8k-test-1018', 'gsm8k-test-1019', 'gsm8k-test-1020', 'gsm8k-test-1021', 'gsm8k-test-1022', 'gsm8k-test-1023', 'gsm8k-test-1024', 'gsm8k-test-1025', 'gsm8k-test-1026', 'gsm8k-test-1027', 'gsm8k-test-1028', 'gsm8k-test-1029', 'gsm8k-test-1030', 'gsm8k-test-1031', 'gsm8k-test-1032', 'gsm8k-test-1033', 'gsm8k-test-1034', 'gsm8k-test-1035', 'gsm8k-test-1036', 'gsm8k-test-1037', 'gsm8k-test-1038', 'gsm8k-test-1039', 'gsm8k-test-1040', 'gsm8k-test-1041', 'gsm8k-test-1042', 'gsm8k-test-1043', 'gsm8k-test-1044', 'gsm8k-test-1045', 'gsm8k-test-1046', 'gsm8k-test-1047', 'gsm8k-test-1048', 'gsm8k-test-1049', 'gsm8k-test-1050', 'gsm8k-test-1051', 'gsm8k-test-1052', 'gsm8k-test-1053', 'gsm8k-test-1054', 'gsm8k-test-1055', 'gsm8k-test-1056', 'gsm8k-test-1057', 'gsm8k-test-1058', 'gsm8k-test-1059', 'gsm8k-test-1060', 'gsm8k-test-1061', 'gsm8k-test-1062', 'gsm8k-test-1063', 'gsm8k-test-1064', 'gsm8k-test-1065', 'gsm8k-test-1066', 'gsm8k-test-1067', 'gsm8k-test-1068', 'gsm8k-test-1069', 'gsm8k-test-1070', 'gsm8k-test-1071', 'gsm8k-test-1072', 'gsm8k-test-1073', 'gsm8k-test-1074', 'gsm8k-test-1075', 'gsm8k-test-1076', 'gsm8k-test-1077', 'gsm8k-test-1078', 'gsm8k-test-1079', 'gsm8k-test-1080', 'gsm8k-test-1081', 'gsm8k-test-1082', 'gsm8k-test-1083', 'gsm8k-test-1084', 'gsm8k-test-1085', 'gsm8k-test-1086', 'gsm8k-test-1087', 'gsm8k-test-1088', 'gsm8k-test-1089', 'gsm8k-test-1090', 'gsm8k-test-1091', 'gsm8k-test-1092', 'gsm8k-test-1093', 'gsm8k-test-1094', 'gsm8k-test-1095', 'gsm8k-test-1096', 'gsm8k-test-1097', 'gsm8k-test-1098', 'gsm8k-test-1099', 'gsm8k-test-1100', 'gsm8k-test-1101', 'gsm8k-test-1102', 'gsm8k-test-1103', 'gsm8k-test-1104', 'gsm8k-test-1105', 'gsm8k-test-1106', 'gsm8k-test-1107', 'gsm8k-test-1108', 'gsm8k-test-1109', 'gsm8k-test-1110', 'gsm8k-test-1111', 'gsm8k-test-1112', 'gsm8k-test-1113', 'gsm8k-test-1114', 'gsm8k-test-1115', 'gsm8k-test-1116', 'gsm8k-test-1117', 'gsm8k-test-1118', 'gsm8k-test-1119', 'gsm8k-test-1120', 'gsm8k-test-1121', 'gsm8k-test-1122', 'gsm8k-test-1123', 'gsm8k-test-1124', 'gsm8k-test-1125', 'gsm8k-test-1126', 'gsm8k-test-1127', 'gsm8k-test-1128', 'gsm8k-test-1129', 'gsm8k-test-1130', 'gsm8k-test-1131', 'gsm8k-test-1132', 'gsm8k-test-1133', 'gsm8k-test-1134', 'gsm8k-test-1135', 'gsm8k-test-1136', 'gsm8k-test-1137', 'gsm8k-test-1138', 'gsm8k-test-1139', 'gsm8k-test-1140', 'gsm8k-test-1141', 'gsm8k-test-1142', 'gsm8k-test-1143', 'gsm8k-test-1144', 'gsm8k-test-1145', 'gsm8k-test-1146', 'gsm8k-test-1147', 'gsm8k-test-1148', 'gsm8k-test-1149', 'gsm8k-test-1150', 'gsm8k-test-1151', 'gsm8k-test-1152', 'gsm8k-test-1153', 'gsm8k-test-1154', 'gsm8k-test-1155', 'gsm8k-test-1156', 'gsm8k-test-1157', 'gsm8k-test-1158', 'gsm8k-test-1159', 'gsm8k-test-1160', 'gsm8k-test-1161', 'gsm8k-test-1162', 'gsm8k-test-1163', 'gsm8k-test-1164', 'gsm8k-test-1165', 'gsm8k-test-1166', 'gsm8k-test-1167', 'gsm8k-test-1168', 'gsm8k-test-1169', 'gsm8k-test-1170', 'gsm8k-test-1171', 'gsm8k-test-1172', 'gsm8k-test-1173', 'gsm8k-test-1174', 'gsm8k-test-1175', 'gsm8k-test-1176', 'gsm8k-test-1177', 'gsm8k-test-1178', 'gsm8k-test-1179', 'gsm8k-test-1180', 'gsm8k-test-1181', 'gsm8k-test-1182', 'gsm8k-test-1183', 'gsm8k-test-1184', 'gsm8k-test-1185', 'gsm8k-test-1186', 'gsm8k-test-1187', 'gsm8k-test-1188', 'gsm8k-test-1189', 'gsm8k-test-1190', 'gsm8k-test-1191', 'gsm8k-test-1192', 'gsm8k-test-1193', 'gsm8k-test-1194', 'gsm8k-test-1195', 'gsm8k-test-1196', 'gsm8k-test-1197', 'gsm8k-test-1198', 'gsm8k-test-1199', 'gsm8k-test-1200', 'gsm8k-test-1201', 'gsm8k-test-1202', 'gsm8k-test-1203', 'gsm8k-test-1204', 'gsm8k-test-1205', 'gsm8k-test-1206', 'gsm8k-test-1207', 'gsm8k-test-1208', 'gsm8k-test-1209', 'gsm8k-test-1210', 'gsm8k-test-1211', 'gsm8k-test-1212', 'gsm8k-test-1213', 'gsm8k-test-1214', 'gsm8k-test-1215', 'gsm8k-test-1216', 'gsm8k-test-1217', 'gsm8k-test-1218', 'gsm8k-test-1219', 'gsm8k-test-1220', 'gsm8k-test-1221', 'gsm8k-test-1222', 'gsm8k-test-1223', 'gsm8k-test-1224', 'gsm8k-test-1225', 'gsm8k-test-1226', 'gsm8k-test-1227', 'gsm8k-test-1228', 'gsm8k-test-1229', 'gsm8k-test-1230', 'gsm8k-test-1231', 'gsm8k-test-1232', 'gsm8k-test-1233', 'gsm8k-test-1234', 'gsm8k-test-1235', 'gsm8k-test-1236', 'gsm8k-test-1237', 'gsm8k-test-1238', 'gsm8k-test-1239', 'gsm8k-test-1240', 'gsm8k-test-1241', 'gsm8k-test-1242', 'gsm8k-test-1243', 'gsm8k-test-1244', 'gsm8k-test-1245', 'gsm8k-test-1246', 'gsm8k-test-1247', 'gsm8k-test-1248', 'gsm8k-test-1249', 'gsm8k-test-1250', 'gsm8k-test-1251', 'gsm8k-test-1252', 'gsm8k-test-1253', 'gsm8k-test-1254', 'gsm8k-test-1255', 'gsm8k-test-1256', 'gsm8k-test-1257', 'gsm8k-test-1258', 'gsm8k-test-1259', 'gsm8k-test-1260', 'gsm8k-test-1261', 'gsm8k-test-1262', 'gsm8k-test-1263', 'gsm8k-test-1264', 'gsm8k-test-1265', 'gsm8k-test-1266', 'gsm8k-test-1267', 'gsm8k-test-1268', 'gsm8k-test-1269', 'gsm8k-test-1270', 'gsm8k-test-1271', 'gsm8k-test-1272', 'gsm8k-test-1273', 'gsm8k-test-1274', 'gsm8k-test-1275', 'gsm8k-test-1276', 'gsm8k-test-1277', 'gsm8k-test-1278', 'gsm8k-test-1279', 'gsm8k-test-1280', 'gsm8k-test-1281', 'gsm8k-test-1282', 'gsm8k-test-1283', 'gsm8k-test-1284', 'gsm8k-test-1285', 'gsm8k-test-1286', 'gsm8k-test-1287', 'gsm8k-test-1288', 'gsm8k-test-1289', 'gsm8k-test-1290', 'gsm8k-test-1291', 'gsm8k-test-1292', 'gsm8k-test-1293', 'gsm8k-test-1294', 'gsm8k-test-1295', 'gsm8k-test-1296', 'gsm8k-test-1297', 'gsm8k-test-1298', 'gsm8k-test-1299', 'gsm8k-test-1300', 'gsm8k-test-1301', 'gsm8k-test-1302', 'gsm8k-test-1303', 'gsm8k-test-1304', 'gsm8k-test-1305', 'gsm8k-test-1306', 'gsm8k-test-1307', 'gsm8k-test-1308', 'gsm8k-test-1309', 'gsm8k-test-1310', 'gsm8k-test-1311', 'gsm8k-test-1312', 'gsm8k-test-1313', 'gsm8k-test-1314', 'gsm8k-test-1315', 'gsm8k-test-1316', 'gsm8k-test-1317', 'gsm8k-test-1318']}}}, 'scores': [{'arm': 'base_model', 'metric': 'accuracy', 'value': 0.7323730098559514, 'evaluator_id': '3bdc9adc3aa8c8f2f9d9e3190001ca7f06d2cf27a204f3b71d02d6983e524423', 'seed': 0, 'predictions_ref': 'predictions-base-model.jsonl'}], 'comparisons': [{'metric': 'accuracy', 'arms': ['gs1_seed0', 'grpo_gs8_seed0']}]}

evaluator_validation

{'status': 'unknown', 'issues': ["Comparison provenance is incomplete; record every declared arm's score and evaluator."], 'comparisons': [{'metric': 'accuracy', 'status': 'unknown', 'arms': ['base_model', 'grpo_gs8_seed0', 'gs1_seed0'], 'evaluator_ids': ['3bdc9adc3aa8c8f2f9d9e3190001ca7f06d2cf27a204f3b71d02d6983e524423'], 'missing_arms': ['grpo_gs8_seed0', 'gs1_seed0']}]}

feasibility

{'status': 'unknown', 'problems': ['Re-projection required: lora_rank changed or was not recorded', 'Re-projection required: dataset changed or was not recorded', 'Re-projection required: steps changed or was not recorded', 'Re-projection required: max_seq_len changed or was not recorded', 'Re-projection required: n_examples changed or was not recorded'], 'wall_seconds': None, 'gpu_hours': None, 'known_gpu_hours': 0, 'phases': {'setup': {'wall_seconds': 49.35325503349304, 'planned_units': 1, 'planned_samples': 0.0, 'gpu_hours': None}, 'training': {'wall_seconds': 5894.319101730982, 'planned_units': 86, 'planned_samples': 5504.0, 'gpu_hours': None}, 'evaluation': {'wall_seconds': 68.7704815864563, 'planned_units': 3957, 'planned_samples': 3957.0, 'gpu_hours': None}, 'recovery': {'wall_seconds': 1800, 'gpu_hours': None}}, 'generation_counts': None, 'generation_applicability': {'check': 'generation_audit', 'status': 'not_applicable', 'enforce': False, 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'features': {'generated_training_data': False}}, 'record': {'config': {'base_model': 'Qwen/Qwen3.5-2B', 'arms': {'grpo_gs8_seed0': {'group_size': 8, 'groups_per_batch': 8}, 'gs1_seed0': {'group_size': 1, 'groups_per_batch': 64, 'kernel': 'word TF-IDF cosine (sklearn defaults, IDF fit on 7473 train questions), w=exp(cos/0.1), self excluded, normalized'}}, 'pilot_steps': 3, 'batch_size': 64, 'max_tokens': 512, 'temperature': 1.0, 'n_eval': 1319}, 'effective_config': {'base_model': 'Qwen/Qwen3.5-2B', 'lora_rank': 32, 'renderer': 'qwen3_5_disable_thinking', 'arms': {'grpo_gs8_seed0': {'group_size': 8, 'groups_per_batch': 8}, 'gs1_seed0': {'group_size': 1, 'groups_per_batch': 64, 'kernel': 'word TF-IDF cosine (sklearn defaults, IDF fit on 7473 train questions), w=exp(cos/0.1), self excluded, normalized'}}, 'steps': 40, 'pilot_steps': 3, 'batch_size': 64, 'max_tokens': 512, 'max_seq_len': 1024, 'temperature': 1.0, 'loss_fn': 'importance_sampling', 'optimizer': {'lr': 4e-05, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08}, 'dataset': 'openai/gsm8k main train (shuffle seed 0)', 'n_eval': 1319, 'eval_decoding': {'temperature': 0, 'max_tokens': 512}, 'tinker': '0.37.0', 'tinker_cookbook': '6daabe0838736557f0e802020b710073eb340287', 'init': 'create_lora_training_client(rank=32) [seed rejected by server] -> save_state -> create_training_client_from_state per arm'}, 'primary_endpoint': 'gsm8k_test_exact_match_accuracy_step40', 'source': 'pilot_results.json (step_seconds per arm-step), eval_base.json', 'measurements': {'setup': {'seconds': 49.35325503349304, 'units': 1, 'planned_units': 1, 'samples': 0, 'gpu_count': None, 'allocation_source': None}, 'training': {'seconds': 411.2315652370453, 'units': 6, 'planned_units': 86, 'samples': 384, 'gpu_count': None, 'allocation_source': None}, 'evaluation': {'seconds': 22.9234938621521, 'units': 1319, 'planned_units': 3957, 'samples': 1319, 'gpu_count': None, 'allocation_source': None}}, 'generation': None, 'recovery': {'seconds': 1800, 'gpu_count': None, 'allocation_source': None}}, 'recorded_at': 1793741581.5690656}

feasibility_required

True

generation_audit_validation

{'status': 'not_applicable', 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'applicability': {'check': 'generation_audit', 'status': 'not_applicable', 'enforce': False, 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'features': {'generated_training_data': False}}}

initialization_provenance

{'version': 1, 'arms': [{'arm': 'grpo_gs8_seed0', 'base_checkpoint': 'Qwen/Qwen3.5-2B', 'init_checkpoint': 'tinker://66bc785a6dd2ae0f72c6707928dfb24b/weights/run_cbb327f6ab-init', 'init_method': 'create_lora_training_client(rank=32; server rejects per-model seeds, so seed 0 not applied) -> save_state -> create_training_client_from_state (fresh optimizer state)', 'adapter_sha256': None, 'evidence_ref': 'out/restoration.json', 'before_first_update': True, 'restore_count': 1, 'optimizer_checks': [{'event': 'restore_from_init_state', 'expected': {'family': 'adam', 'lr': 4e-05, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'state': 'fresh'}, 'actual': {'type': 'adamw'}, 'source': 'TrainingClient.get_info().optimizer_config (server)', 'verified': None, 'note': "server reports optimizer family only ({'type':'adamw'}); lr/betas/eps are per-call AdamParams not echoed by server, so config match unverified"}], 'initial_outputs': {'fixed_batch': 'first 8 GSM8K train rows (question+answer), cross_entropy forward', 'fixed_batch_sha256': '48e2999547b6e6029b0feb6658a241a32ce80f73ec1ce8407285769808f0abee', 'logprobs_sha256': '5ac755852ee024e0bbe1b3d178b57520bf33a99d2b08981579fb8c409d173e87', 'max_abs_diff_vs_init_client': 0.0, 'init_client_logprobs_sha256': '5ac755852ee024e0bbe1b3d178b57520bf33a99d2b08981579fb8c409d173e87'}, 'server_info': {'model_name': 'Qwen/Qwen3.5-2B', 'lora_rank': 32, 'is_lora': True, 'optimizer_config': {'type': 'adamw'}}}, {'arm': 'gs1_seed0', 'base_checkpoint': 'Qwen/Qwen3.5-2B', 'init_checkpoint': 'tinker://66bc785a6dd2ae0f72c6707928dfb24b/weights/run_cbb327f6ab-init', 'init_method': 'create_lora_training_client(rank=32; server rejects per-model seeds, so seed 0 not applied) -> save_state -> create_training_client_from_state (fresh optimizer state)', 'adapter_sha256': None, 'evidence_ref': 'out/restoration.json', 'before_first_update': True, 'restore_count': 1, 'optimizer_checks': [{'event': 'restore_from_init_state', 'expected': {'family': 'adam', 'lr': 4e-05, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'state': 'fresh'}, 'actual': {'type': 'adamw'}, 'source': 'TrainingClient.get_info().optimizer_config (server)', 'verified': None, 'note': "server reports optimizer family only ({'type':'adamw'}); lr/betas/eps are per-call AdamParams not echoed by server, so config match unverified"}], 'initial_outputs': {'fixed_batch': 'first 8 GSM8K train rows (question+answer), cross_entropy forward', 'fixed_batch_sha256': '48e2999547b6e6029b0feb6658a241a32ce80f73ec1ce8407285769808f0abee', 'logprobs_sha256': '5ac755852ee024e0bbe1b3d178b57520bf33a99d2b08981579fb8c409d173e87', 'max_abs_diff_vs_init_client': 0.0, 'init_client_logprobs_sha256': '5ac755852ee024e0bbe1b3d178b57520bf33a99d2b08981579fb8c409d173e87'}, 'server_info': {'model_name': 'Qwen/Qwen3.5-2B', 'lora_rank': 32, 'is_lora': True, 'optimizer_config': {'type': 'adamw'}}}]}

launch

{'experiment_id': 'exp_6ba12d6b47', 'context_status': 'snapshotted', 'superseded': False, 'mode': 'ordinary', 'purpose': 'Execute the selected saved experiment version.'}

launch_instructions

{'name': 'launch-instructions-ea7286a5cb52.md', 'sha256': 'ea7286a5cb526c50069b067713363a61eb22dcdfd226832ce0633fd542133d83', 'url': 'https://ta-01m6fva5db548f5skm214fepcr-8000-oah9l7x00jpo7186eb6cix5v0.w.modal.host/api/runs/run_cbb327f6ab/evidence/ea7286a5cb526c50069b067713363a61eb22dcdfd226832ce0633fd542133d83'}

reported_outcome_reason

incomplete_execution

reported_verdict

inconclusive

restoration_validation

{'status': 'unverified', 'required': True, 'checks': [{'check': 'grpo_gs8_seed0', 'status': 'unverified', 'reason': 'grpo_gs8_seed0: missing or duplicate arm evidence.'}, {'check': 'gs1_seed0', 'status': 'unverified', 'reason': 'gs1_seed0: missing or duplicate arm evidence.'}], 'reason': 'grpo_gs8_seed0: missing or duplicate arm evidence.', 'applicability': 'required'}