🔬Modal Research Exchange

{
  "activation_gate": {
    "applicable": true,
    "checks": [
      {
        "max": 0.3,
        "measures": "fraction of loss tokens masked by the DPPO divergence rule on the second minibatch update, mean over pilot steps",
        "metric": "dppo_masked_token_fraction",
        "min": 0.001
      },
      {
        "measures": "90th percentile of |p_current - p_sampler| over loss tokens at the second update",
        "metric": "policy_drift_abs_p_q90",
        "min": 0.005
      },
      {
        "measures": "fraction of tokens clipped by the eps=0.2 ratio clip in the baseline loss on the same pilot batches (confirms the regime is off-policy for the baseline too)",
        "metric": "grpo_clip_fraction",
        "min": 0.0005
      },
      {
        "max": 2,
        "measures": "optim_step calls per rollout batch as logged",
        "metric": "executed_updates_per_batch",
        "min": 2
      },
      {
        "measures": "model name reported at client creation",
        "metric": "executed_base_model",
        "one_of": [
          "Qwen/Qwen3.5-2B"
        ]
      }
    ],
    "reason": "DPPO only differs from GRPO on tokens whose divergence exceeds the trust region. A 5-step pilot of the treatment arm, with 2 updates per batch, must show that the mask fires on a nontrivial but not dominant share of tokens and that the policy actually drifts. Executed config values must match the plan."
  },
  "arms": [
    {
      "loss": "grpo_clip_eps0.2",
      "lr": 2e-05,
      "name": "grpo_seed0",
      "seed": 0,
      "updates_per_batch": 2
    },
    {
      "loss": "dppo_divergence_mask",
      "lr": 2e-05,
      "name": "dppo_seed0",
      "seed": 0,
      "updates_per_batch": 2
    }
  ],
  "base_model": "Qwen/Qwen3.5-2B",
  "baseline": "GRPO clipped-ratio loss (eps=0.2) through the same custom loss path. Identical rollouts per step, 2 minibatch updates per batch, lr, rank, seed, data order and eval.",
  "batch_size": 256,
  "budget": {
    "gpu_hours": 2.0,
    "max_wall_minutes": 120
  },
  "command": "Custom tinker SDK loop; the loss function is the custom part. Sketch:\n\nimport os, tinker\nsc = tinker.ServiceClient(base_url=os.environ['TINKER_BASE_URL'])\ntc = sc.create_lora_training_client(base_model='Qwen/Qwen3.5-2B', rank=32)\n\n# Rollouts: reuse tinker_cookbook.recipes.math_rl GSM8K env and renderer.\n# group_size=8, groups_per_batch=32, max_tokens=512, seed=0.\nfor step in range(50):\n    sampler = tc.save_weights_and_get_sampling_client(f's{step}')\n    batch = rollout(sampler, gsm8k_train_prompts[step], group_size=8)  # stores sampler logprobs as old_lp\n    adv = group_mean_center(batch.rewards)\n    for mb in split(batch, 2):\n        tc.forward_backward_custom(mb.data, make_loss(LOSS, old_lp=mb.old_lp, adv=mb.adv)).result()\n        tc.optim_step(tinker.AdamParams(learning_rate=2e-5)).result()\n\n# LOSS='grpo': -min(r*A, clip(r, 0.8, 1.2)*A)\n# LOSS='dppo': -(r*A) * keep_mask(div(p, p_old) <= delta), with div and delta pinned from the paper code\n# Final eval: greedy decoding on openai/gsm8k test (1319 items), numeric exact match.",
  "correctness_preflight": {
    "paths": [
      {
        "baseline_revision": "UNRESOLVED: pin tinker-cookbook commit whose stock ppo/importance-sampling loss defines GRPO clipping",
        "id": "grpo_dppo_custom_loss",
        "kind": "loss",
        "reference_revision": "UNRESOLVED: pin commit of the official DPPO implementation for arXiv 2602.04879",
        "tolerances": {
          "baseline_gradient": {
            "atol": 1e-05,
            "min_cosine": 0.9999,
            "rtol": 0.001
          },
          "baseline_loss": {
            "atol": 1e-05,
            "rtol": 0.0001
          },
          "reference_gradient": {
            "atol": 1e-05,
            "min_cosine": 0.9999,
            "rtol": 0.001
          },
          "reference_loss": {
            "atol": 1e-05,
            "rtol": 0.0001
          }
        }
      }
    ],
    "required": true,
    "status": "unresolved: DPPO reference code revision and exact divergence/threshold not yet pinned (paper digest failed); no full sweep until resolved"
  },
  "dataset": {
    "hf_path": "openai/gsm8k",
    "n_examples": 1600,
    "name": "gsm8k",
    "split": "main/train"
  },
  "decision_rule": {
    "alpha": 0.05,
    "criteria": [
      {
        "bounds": [
          0,
          1
        ],
        "id": "primary",
        "margin": 0.0,
        "metric": "accuracy",
        "weights": {
          "baseline": -1,
          "treatment": 1
        }
      }
    ],
    "estimand": "Difference in greedy GSM8K test exact-match accuracy, DPPO minus GRPO, for these fixed final checkpoints of Qwen3.5-2B LoRA; larger is better",
    "method": "paired_hoeffding",
    "seeds": [
      0
    ],
    "unit": "fixed_checkpoint_items",
    "version": 1
  },
  "eval": {
    "aggregation": null,
    "benchmark": "GSM8K test (openai/gsm8k main/test)",
    "metric": "greedy exact-match accuracy",
    "n_examples": 1319
  },
  "experiment_features": {
    "adaptive_selection": false,
    "checkpoint_restore": false,
    "custom_loss": true,
    "custom_optimizer": false,
    "generated_training_data": false,
    "heldout_loss": false,
    "learned_generator": false,
    "paired_comparison": true,
    "paired_training_arms": true,
    "warmup_adaptation": false
  },
  "feasibility": null,
  "feasibility_check": {
    "gpu_hours": null,
    "known_gpu_hours": 0,
    "phases": {},
    "problems": [
      "Measured feasibility not recorded"
    ],
    "status": "unknown",
    "wall_seconds": null
  },
  "feasibility_required": true,
  "generation_audit": {
    "answer_comparison": "numeric_exact",
    "max_eval_overlaps": 0,
    "max_label_errors": 0,
    "min_sample_size": 100,
    "required": false
  },
  "hypothesis": "On Qwen3.5-2B LoRA RL on GSM8K with 2 policy updates per rollout batch, the DPPO loss reaches higher greedy exact-match accuracy on the 1319-item GSM8K test set than the GRPO clipped-ratio loss. The lower bound of the paired 95% CI on DPPO minus GRPO must be above 0.",
  "lora_rank": 32,
  "max_seq_len": 1024,
  "method": "The requested Qwen3.5-0.8B is not served, so we use the closest served small model, Qwen/Qwen3.5-2B.\n\nBoth arms run the same client-side RL loop: the tinker-cookbook math_rl GSM8K env, renderer, prompts, reward and data order. Each step samples groups_per_batch=32 GSM8K train prompts with group_size=8 rollouts at temperature 1.0 and max_tokens=512. Advantages are group-mean-centered (GRPO).\n\nThe rollout batch is split into 2 minibatches. That gives 2 optim_steps per batch, so the policy drifts from the sampling policy and the trust region can act. With one update per fresh batch, neither mask nor clip fires. Both losses are computed in one forked custom loss path through forward_backward_custom, using the sampler logprobs as p_old.\n\nThe arms differ in exactly one switch:\n- Baseline (GRPO): token loss -min(r*A, clip(r, 1-0.2, 1+0.2)*A).\n- Treatment (DPPO, arXiv 2602.04879): token loss -r*A, with tokens masked out when the divergence between the current and sampling policy exceeds the threshold delta in the direction the advantage pushes. The divergence is estimated from the token probabilities, e.g. |p - p_old|, rather than the single-sample ratio.\n\nTriage only received a truncated digest of the paper. The exact divergence estimator, delta and masking direction must be pinned from the paper's released code before any training. Until then the correctness preflight is unresolved and no full sweep runs.\n\nTraining: LoRA rank 32, AdamW lr 2e-5, 50 rollout steps (100 optim steps) per arm, seed 0. Both arms start from the same base weights.\n\nEvaluation: the final checkpoint of each arm is scored with greedy decoding on all 1319 GSM8K test items, using the cookbook's numeric answer grader.",
  "optimizer": {
    "lr": 2e-05,
    "name": "adam"
  },
  "primary_endpoint": "gsm8k_test_greedy_exact_match_accuracy@final_checkpoint (1319 items)",
  "restoration_preflight": {
    "applicable": true,
    "fixed_batch_required": true,
    "groups": [
      {
        "arms": [
          {
            "name": "grpo_seed0",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 2e-05
            }
          },
          {
            "name": "dppo_seed0",
            "optimizer": "adam",
            "optimizer_config": {
              "beta1": 0.9,
              "beta2": 0.95,
              "eps": 1e-08,
              "learning_rate": 2e-05
            }
          }
        ]
      }
    ],
    "reason": "Both arms are LoRA-trained from the same base. Before the first update, verify identical initial state: compare the logprobs on a fixed tokenized GSM8K batch across both freshly created clients, which must be bitwise or within 1e-6. Also verify the active optimizer is AdamW with the planned config after creation. Server identity cannot be introspected and stays unverified."
  },
  "steps": 100,
  "success_criterion": "On the 1319 GSM8K test items, the paired 95% Hoeffding CI for DPPO accuracy minus GRPO accuracy has a lower bound above 0.0, roughly a 7.5-point gap at this n. A smaller positive gap is reported as inconclusive, not validated.",
  "title": "DPPO divergence-masked loss vs GRPO ratio-clipped loss, Qwen3.5-2B GSM8K LoRA RL",
  "training_type": "rl"
}

[
  {
    "arm": "shared_init",
    "step": 0,
    "uri": "tinker://21e32976f18f971463410adff4e364e4/weights/rx19badd842c_init"
  },
  {
    "arm": "dppo_pilot",
    "step": 5,
    "uri": "tinker://8004a92b-0b51-4ec0-a996-ab27ef0a7254:train:0/sampler_weights/dppo_pilot_final_s5"
  },
  {
    "arm": "dppo_pilot",
    "step": 5,
    "uri": "tinker://8004a92b-0b51-4ec0-a996-ab27ef0a7254:train:0/weights/dppo_pilot_state_s5"
  }
]

{
  "base": {
    "accuracy": 0.7035633055344959
  }
}

base_accuracy

0.7036

base_n_items

1,319

base_n_correct

928

pilot_steps

5

pilot_dppo_masked_token_fraction_mb2_mean

0

pilot_policy_drift_abs_p_q90_mb2_mean

0.01052

pilot_grpo_clip_fraction_mb2_mean

0.0004569

pilot_executed_updates_per_batch

2

pilot_reward_mean_last

0.5645

pilot_train_correct_mean_mean

0.6141

comparison_steps_completed

0

• checkpoint · shared_init · step 0 · tinker://21e32976f18f971463410adff4e364e4/weights/rx19badd842c_init
• checkpoint · dppo_pilot · step 5 · tinker://8004a92b-0b51-4ec0-a996-ab27ef0a7254:train:0/sampler_weights/dppo_pilot_final_s5
• checkpoint · dppo_pilot · step 5 · tinker://8004a92b-0b51-4ec0-a996-ab27ef0a7254:train:0/weights/dppo_pilot_state_s5
• instructions · launch-instructions.md sha256 5ef650264f18
• script · rx.py sha256 cb8269a8a998
• script · losses.py sha256 36e744853b41
• script · preflight_local.py sha256 7d48af2a5992
• effective_config · effective-config.json
• effective_config · spec_effective.json sha256 08a42fc27acb
• code_revision · code-revision.json
• metrics · preflight.json sha256 2d35b9fe3018
• metrics · preflight_fixture.json sha256 aff0d4c84ff1
• metrics · validator_spec.json
• metrics · validator_spec_effective.json
• metrics · pilot-dppo-steps.jsonl sha256 ee68c6b8951c
• metrics · pilot-restore-init.json
• metrics · evaluator.json
• metrics · eval-base-summary.json
• predictions · predictions_base.jsonl sha256 098bcd66552c

1. Nov 3, 21:30 UTC · failed · On shared Spindle with Qwen/Qwen3.5-2B (the closest served model to Qwen3.5-0.8B), I pinned DPPO Binary-TV (Stable-RL@c6af1d1, delta=0.15) and Spindle's stock ppo as references, passed loss+gradient …

correctness_preflight

{'required': True, 'paths': [{'id': 'grpo_dppo_custom_loss', 'fixed_input': {'sha256': 'aff0d4c84ff104219bf365a13dcb0de1c1cfc59b833ac72cb357c46e29a83f13', 'description': 'preflight_fixture.json: 16 ragged seqs (len 40-96), prompt-masked, stale current logprobs = sampler logprobs + N(0,0.6) noise (seed 0) so GRPO clip (35%) and DPPO mask (18.6%) both fire; one zero-advantage seq. Probe: code/preflight_local.py executes pinned source functions via ast extraction'}, 'baseline_revision': "modal-projects/spindle@e58ebef130b4cd02de6880e62d27ec358ff7132c:src/spindle/backends/megatron_runtime/common/forward_backward.py::_loss(loss_name='ppo', clip 0.8/1.2) — Spindle's stock Tinker 'ppo' loss (tinker-cookbook 0.5.9 has no client-side ppo; loss_fn='ppo' is server stock)", 'reference_revision': "sail-sg/Stable-RL@c6af1d119c0fe30d16dd9a837d35fdf695acd226:verl/trainer/ppo/core_algos.py::compute_policy_loss_with_mask(loss_mode='dppo_binary_tv', clip_ratio=0.15, clip_ratio_c=10)", 'implementation_revision': 'code/losses.py sha256 36e744853b410170a874b80e523914ec5b0f04fa7781d96b9d04abd0e10271d3', 'precision': 'float32 CPU torch (Tinker logprobs dtype); errors computed in float64', 'normalization': 'sum over loss-mask tokens (Tinker forward_backward convention); reference agg_loss token-mean with batch_num_tokens=1, dp_size=1 == masked sum; ours per-datum unpacked list vs reference padded [B,T] layout (baseline: spindle per-sequence flattened rows)', 'errors': {'baseline_loss': {'max_abs_error': 6.103515625e-05, 'reference_max_abs': 429.4608154296875, 'relative_l2_error': 1.4212043114790956e-07, 'cosine_similarity': 1.0}, 'baseline_gradient': {'max_abs_error': 0.0, 'reference_max_abs': 8.006952285766602, 'relative_l2_error': 0.0, 'cosine_similarity': 1.0}, 'reference_loss': {'max_abs_error': 3.0517578125e-05, 'reference_max_abs': 420.54302978515625, 'relative_l2_error': 7.25670762884611e-08, 'cosine_similarity': 1.0}, 'reference_gradient': {'max_abs_error': 0.0, 'reference_max_abs': 8.006952285766602, 'relative_l2_error': 0.0, 'cosine_similarity': 1.0}}}]}

correctness_validation

{'status': 'unverified', 'required': True, 'checks': [{'path': 'grpo_dppo_custom_loss', 'check': 'baseline_loss', 'status': 'passed', 'tolerance': {'atol': 1e-05, 'rtol': 0.0001}, 'measured': {'max_abs_error': 6.103515625e-05, 'reference_max_abs': 429.4608154296875, 'relative_l2_error': 1.4212043114790956e-07, 'cosine_similarity': 1.0}}, {'path': 'grpo_dppo_custom_loss', 'check': 'baseline_gradient', 'status': 'passed', 'tolerance': {'atol': 1e-05, 'rtol': 0.001, 'min_cosine': 0.9999}, 'measured': {'max_abs_error': 0.0, 'reference_max_abs': 8.006952285766602, 'relative_l2_error': 0.0, 'cosine_similarity': 1.0}}, {'path': 'grpo_dppo_custom_loss', 'check': 'reference_loss', 'status': 'passed', 'tolerance': {'atol': 1e-05, 'rtol': 0.0001}, 'measured': {'max_abs_error': 3.0517578125e-05, 'reference_max_abs': 420.54302978515625, 'relative_l2_error': 7.25670762884611e-08, 'cosine_similarity': 1.0}}, {'path': 'grpo_dppo_custom_loss', 'check': 'reference_gradient', 'status': 'passed', 'tolerance': {'atol': 1e-05, 'rtol': 0.001, 'min_cosine': 0.9999}, 'measured': {'max_abs_error': 0.0, 'reference_max_abs': 8.006952285766602, 'relative_l2_error': 0.0, 'cosine_similarity': 1.0}}], 'reason': 'grpo_dppo_custom_loss: matching reference_revision; grpo_dppo_custom_loss: matching baseline_revision', 'evidence_source': 'executor report; remote execution is not attested', 'applicability': 'required', 'reported_verdict': 'inconclusive'}

decision_validation

{'status': 'uncertain', 'issues': ['Decision evidence was not recorded.'], 'criteria': [], 'outcome_reason': 'insufficient_evidence', 'unit': 'fixed_checkpoint_items', 'n_seeds': 1, 'alpha': 0.05, 'method': 'paired_hoeffding', 'applicability': {'check': 'decision_rule', 'status': 'required', 'enforce': True, 'reasons': ['experiment_features: paired_comparison is true'], 'features': {'paired_comparison': True, 'adaptive_selection': False}}}

evaluation

{'version': 1, 'evaluators': {'b9aa01a393125755d46944c706831466ffc955df7a4cad35f711050ce7ef500f': {'code_sha256': '54111ffd3f504c0522bf7e7acfc8a9fd98cd492505f775ee05a91f933844e108', 'prompt_template': "tinker_cookbook 0.5.9 renderer qwen3_5_disable_thinking (Qwen/Qwen3.5-2B tokenizer chat template) build_generation_prompt(MathEnv.standard_fewshot_prefix() + [user: question + ' Write your answer in \\\\boxed{} format.']); code/rx.py evaluate()", 'decoding': {'temperature': 0.0, 'max_tokens': 512, 'stop': [248046], 'num_samples': 1}, 'parser': 'renderer.parse_response -> renderers.get_text_content -> tinker_cookbook.recipes.math_rl.math_env.MathEnv.check_answer (math_grading.extract_boxed + safe_grade grader=sympy timeout=1.0); reference = extract_gsm8k_final_answer(answer); exceptions scored 0', 'items': {'ids': ['gsm8k_test_0', 'gsm8k_test_1', 'gsm8k_test_2', 'gsm8k_test_3', 'gsm8k_test_4', 'gsm8k_test_5', 'gsm8k_test_6', 'gsm8k_test_7', 'gsm8k_test_8', 'gsm8k_test_9', 'gsm8k_test_10', 'gsm8k_test_11', 'gsm8k_test_12', 'gsm8k_test_13', 'gsm8k_test_14', 'gsm8k_test_15', 'gsm8k_test_16', 'gsm8k_test_17', 'gsm8k_test_18', 'gsm8k_test_19', 'gsm8k_test_20', 'gsm8k_test_21', 'gsm8k_test_22', 'gsm8k_test_23', 'gsm8k_test_24', 'gsm8k_test_25', 'gsm8k_test_26', 'gsm8k_test_27', 'gsm8k_test_28', 'gsm8k_test_29', 'gsm8k_test_30', 'gsm8k_test_31', 'gsm8k_test_32', 'gsm8k_test_33', 'gsm8k_test_34', 'gsm8k_test_35', 'gsm8k_test_36', 'gsm8k_test_37', 'gsm8k_test_38', 'gsm8k_test_39', 'gsm8k_test_40', 'gsm8k_test_41', 'gsm8k_test_42', 'gsm8k_test_43', 'gsm8k_test_44', 'gsm8k_test_45', 'gsm8k_test_46', 'gsm8k_test_47', 'gsm8k_test_48', 'gsm8k_test_49', 'gsm8k_test_50', 'gsm8k_test_51', 'gsm8k_test_52', 'gsm8k_test_53', 'gsm8k_test_54', 'gsm8k_test_55', 'gsm8k_test_56', 'gsm8k_test_57', 'gsm8k_test_58', 'gsm8k_test_59', 'gsm8k_test_60', 'gsm8k_test_61', 'gsm8k_test_62', 'gsm8k_test_63', 'gsm8k_test_64', 'gsm8k_test_65', 'gsm8k_test_66', 'gsm8k_test_67', 'gsm8k_test_68', 'gsm8k_test_69', 'gsm8k_test_70', 'gsm8k_test_71', 'gsm8k_test_72', 'gsm8k_test_73', 'gsm8k_test_74', 'gsm8k_test_75', 'gsm8k_test_76', 'gsm8k_test_77', 'gsm8k_test_78', 'gsm8k_test_79', 'gsm8k_test_80', 'gsm8k_test_81', 'gsm8k_test_82', 'gsm8k_test_83', 'gsm8k_test_84', 'gsm8k_test_85', 'gsm8k_test_86', 'gsm8k_test_87', 'gsm8k_test_88', 'gsm8k_test_89', 'gsm8k_test_90', 'gsm8k_test_91', 'gsm8k_test_92', 'gsm8k_test_93', 'gsm8k_test_94', 'gsm8k_test_95', 'gsm8k_test_96', 'gsm8k_test_97', 'gsm8k_test_98', 'gsm8k_test_99', 'gsm8k_test_100', 'gsm8k_test_101', 'gsm8k_test_102', 'gsm8k_test_103', 'gsm8k_test_104', 'gsm8k_test_105', 'gsm8k_test_106', 'gsm8k_test_107', 'gsm8k_test_108', 'gsm8k_test_109', 'gsm8k_test_110', 'gsm8k_test_111', 'gsm8k_test_112', 'gsm8k_test_113', 'gsm8k_test_114', 'gsm8k_test_115', 'gsm8k_test_116', 'gsm8k_test_117', 'gsm8k_test_118', 'gsm8k_test_119', 'gsm8k_test_120', 'gsm8k_test_121', 'gsm8k_test_122', 'gsm8k_test_123', 'gsm8k_test_124', 'gsm8k_test_125', 'gsm8k_test_126', 'gsm8k_test_127', 'gsm8k_test_128', 'gsm8k_test_129', 'gsm8k_test_130', 'gsm8k_test_131', 'gsm8k_test_132', 'gsm8k_test_133', 'gsm8k_test_134', 'gsm8k_test_135', 'gsm8k_test_136', 'gsm8k_test_137', 'gsm8k_test_138', 'gsm8k_test_139', 'gsm8k_test_140', 'gsm8k_test_141', 'gsm8k_test_142', 'gsm8k_test_143', 'gsm8k_test_144', 'gsm8k_test_145', 'gsm8k_test_146', 'gsm8k_test_147', 'gsm8k_test_148', 'gsm8k_test_149', 'gsm8k_test_150', 'gsm8k_test_151', 'gsm8k_test_152', 'gsm8k_test_153', 'gsm8k_test_154', 'gsm8k_test_155', 'gsm8k_test_156', 'gsm8k_test_157', 'gsm8k_test_158', 'gsm8k_test_159', 'gsm8k_test_160', 'gsm8k_test_161', 'gsm8k_test_162', 'gsm8k_test_163', 'gsm8k_test_164', 'gsm8k_test_165', 'gsm8k_test_166', 'gsm8k_test_167', 'gsm8k_test_168', 'gsm8k_test_169', 'gsm8k_test_170', 'gsm8k_test_171', 'gsm8k_test_172', 'gsm8k_test_173', 'gsm8k_test_174', 'gsm8k_test_175', 'gsm8k_test_176', 'gsm8k_test_177', 'gsm8k_test_178', 'gsm8k_test_179', 'gsm8k_test_180', 'gsm8k_test_181', 'gsm8k_test_182', 'gsm8k_test_183', 'gsm8k_test_184', 'gsm8k_test_185', 'gsm8k_test_186', 'gsm8k_test_187', 'gsm8k_test_188', 'gsm8k_test_189', 'gsm8k_test_190', 'gsm8k_test_191', 'gsm8k_test_192', 'gsm8k_test_193', 'gsm8k_test_194', 'gsm8k_test_195', 'gsm8k_test_196', 'gsm8k_test_197', 'gsm8k_test_198', 'gsm8k_test_199', 'gsm8k_test_200', 'gsm8k_test_201', 'gsm8k_test_202', 'gsm8k_test_203', 'gsm8k_test_204', 'gsm8k_test_205', 'gsm8k_test_206', 'gsm8k_test_207', 'gsm8k_test_208', 'gsm8k_test_209', 'gsm8k_test_210', 'gsm8k_test_211', 'gsm8k_test_212', 'gsm8k_test_213', 'gsm8k_test_214', 'gsm8k_test_215', 'gsm8k_test_216', 'gsm8k_test_217', 'gsm8k_test_218', 'gsm8k_test_219', 'gsm8k_test_220', 'gsm8k_test_221', 'gsm8k_test_222', 'gsm8k_test_223', 'gsm8k_test_224', 'gsm8k_test_225', 'gsm8k_test_226', 'gsm8k_test_227', 'gsm8k_test_228', 'gsm8k_test_229', 'gsm8k_test_230', 'gsm8k_test_231', 'gsm8k_test_232', 'gsm8k_test_233', 'gsm8k_test_234', 'gsm8k_test_235', 'gsm8k_test_236', 'gsm8k_test_237', 'gsm8k_test_238', 'gsm8k_test_239', 'gsm8k_test_240', 'gsm8k_test_241', 'gsm8k_test_242', 'gsm8k_test_243', 'gsm8k_test_244', 'gsm8k_test_245', 'gsm8k_test_246', 'gsm8k_test_247', 'gsm8k_test_248', 'gsm8k_test_249', 'gsm8k_test_250', 'gsm8k_test_251', 'gsm8k_test_252', 'gsm8k_test_253', 'gsm8k_test_254', 'gsm8k_test_255', 'gsm8k_test_256', 'gsm8k_test_257', 'gsm8k_test_258', 'gsm8k_test_259', 'gsm8k_test_260', 'gsm8k_test_261', 'gsm8k_test_262', 'gsm8k_test_263', 'gsm8k_test_264', 'gsm8k_test_265', 'gsm8k_test_266', 'gsm8k_test_267', 'gsm8k_test_268', 'gsm8k_test_269', 'gsm8k_test_270', 'gsm8k_test_271', 'gsm8k_test_272', 'gsm8k_test_273', 'gsm8k_test_274', 'gsm8k_test_275', 'gsm8k_test_276', 'gsm8k_test_277', 'gsm8k_test_278', 'gsm8k_test_279', 'gsm8k_test_280', 'gsm8k_test_281', 'gsm8k_test_282', 'gsm8k_test_283', 'gsm8k_test_284', 'gsm8k_test_285', 'gsm8k_test_286', 'gsm8k_test_287', 'gsm8k_test_288', 'gsm8k_test_289', 'gsm8k_test_290', 'gsm8k_test_291', 'gsm8k_test_292', 'gsm8k_test_293', 'gsm8k_test_294', 'gsm8k_test_295', 'gsm8k_test_296', 'gsm8k_test_297', 'gsm8k_test_298', 'gsm8k_test_299', 'gsm8k_test_300', 'gsm8k_test_301', 'gsm8k_test_302', 'gsm8k_test_303', 'gsm8k_test_304', 'gsm8k_test_305', 'gsm8k_test_306', 'gsm8k_test_307', 'gsm8k_test_308', 'gsm8k_test_309', 'gsm8k_test_310', 'gsm8k_test_311', 'gsm8k_test_312', 'gsm8k_test_313', 'gsm8k_test_314', 'gsm8k_test_315', 'gsm8k_test_316', 'gsm8k_test_317', 'gsm8k_test_318', 'gsm8k_test_319', 'gsm8k_test_320', 'gsm8k_test_321', 'gsm8k_test_322', 'gsm8k_test_323', 'gsm8k_test_324', 'gsm8k_test_325', 'gsm8k_test_326', 'gsm8k_test_327', 'gsm8k_test_328', 'gsm8k_test_329', 'gsm8k_test_330', 'gsm8k_test_331', 'gsm8k_test_332', 'gsm8k_test_333', 'gsm8k_test_334', 'gsm8k_test_335', 'gsm8k_test_336', 'gsm8k_test_337', 'gsm8k_test_338', 'gsm8k_test_339', 'gsm8k_test_340', 'gsm8k_test_341', 'gsm8k_test_342', 'gsm8k_test_343', 'gsm8k_test_344', 'gsm8k_test_345', 'gsm8k_test_346', 'gsm8k_test_347', 'gsm8k_test_348', 'gsm8k_test_349', 'gsm8k_test_350', 'gsm8k_test_351', 'gsm8k_test_352', 'gsm8k_test_353', 'gsm8k_test_354', 'gsm8k_test_355', 'gsm8k_test_356', 'gsm8k_test_357', 'gsm8k_test_358', 'gsm8k_test_359', 'gsm8k_test_360', 'gsm8k_test_361', 'gsm8k_test_362', 'gsm8k_test_363', 'gsm8k_test_364', 'gsm8k_test_365', 'gsm8k_test_366', 'gsm8k_test_367', 'gsm8k_test_368', 'gsm8k_test_369', 'gsm8k_test_370', 'gsm8k_test_371', 'gsm8k_test_372', 'gsm8k_test_373', 'gsm8k_test_374', 'gsm8k_test_375', 'gsm8k_test_376', 'gsm8k_test_377', 'gsm8k_test_378', 'gsm8k_test_379', 'gsm8k_test_380', 'gsm8k_test_381', 'gsm8k_test_382', 'gsm8k_test_383', 'gsm8k_test_384', 'gsm8k_test_385', 'gsm8k_test_386', 'gsm8k_test_387', 'gsm8k_test_388', 'gsm8k_test_389', 'gsm8k_test_390', 'gsm8k_test_391', 'gsm8k_test_392', 'gsm8k_test_393', 'gsm8k_test_394', 'gsm8k_test_395', 'gsm8k_test_396', 'gsm8k_test_397', 'gsm8k_test_398', 'gsm8k_test_399', 'gsm8k_test_400', 'gsm8k_test_401', 'gsm8k_test_402', 'gsm8k_test_403', 'gsm8k_test_404', 'gsm8k_test_405', 'gsm8k_test_406', 'gsm8k_test_407', 'gsm8k_test_408', 'gsm8k_test_409', 'gsm8k_test_410', 'gsm8k_test_411', 'gsm8k_test_412', 'gsm8k_test_413', 'gsm8k_test_414', 'gsm8k_test_415', 'gsm8k_test_416', 'gsm8k_test_417', 'gsm8k_test_418', 'gsm8k_test_419', 'gsm8k_test_420', 'gsm8k_test_421', 'gsm8k_test_422', 'gsm8k_test_423', 'gsm8k_test_424', 'gsm8k_test_425', 'gsm8k_test_426', 'gsm8k_test_427', 'gsm8k_test_428', 'gsm8k_test_429', 'gsm8k_test_430', 'gsm8k_test_431', 'gsm8k_test_432', 'gsm8k_test_433', 'gsm8k_test_434', 'gsm8k_test_435', 'gsm8k_test_436', 'gsm8k_test_437', 'gsm8k_test_438', 'gsm8k_test_439', 'gsm8k_test_440', 'gsm8k_test_441', 'gsm8k_test_442', 'gsm8k_test_443', 'gsm8k_test_444', 'gsm8k_test_445', 'gsm8k_test_446', 'gsm8k_test_447', 'gsm8k_test_448', 'gsm8k_test_449', 'gsm8k_test_450', 'gsm8k_test_451', 'gsm8k_test_452', 'gsm8k_test_453', 'gsm8k_test_454', 'gsm8k_test_455', 'gsm8k_test_456', 'gsm8k_test_457', 'gsm8k_test_458', 'gsm8k_test_459', 'gsm8k_test_460', 'gsm8k_test_461', 'gsm8k_test_462', 'gsm8k_test_463', 'gsm8k_test_464', 'gsm8k_test_465', 'gsm8k_test_466', 'gsm8k_test_467', 'gsm8k_test_468', 'gsm8k_test_469', 'gsm8k_test_470', 'gsm8k_test_471', 'gsm8k_test_472', 'gsm8k_test_473', 'gsm8k_test_474', 'gsm8k_test_475', 'gsm8k_test_476', 'gsm8k_test_477', 'gsm8k_test_478', 'gsm8k_test_479', 'gsm8k_test_480', 'gsm8k_test_481', 'gsm8k_test_482', 'gsm8k_test_483', 'gsm8k_test_484', 'gsm8k_test_485', 'gsm8k_test_486', 'gsm8k_test_487', 'gsm8k_test_488', 'gsm8k_test_489', 'gsm8k_test_490', 'gsm8k_test_491', 'gsm8k_test_492', 'gsm8k_test_493', 'gsm8k_test_494', 'gsm8k_test_495', 'gsm8k_test_496', 'gsm8k_test_497', 'gsm8k_test_498', 'gsm8k_test_499', 'gsm8k_test_500', 'gsm8k_test_501', 'gsm8k_test_502', 'gsm8k_test_503', 'gsm8k_test_504', 'gsm8k_test_505', 'gsm8k_test_506', 'gsm8k_test_507', 'gsm8k_test_508', 'gsm8k_test_509', 'gsm8k_test_510', 'gsm8k_test_511', 'gsm8k_test_512', 'gsm8k_test_513', 'gsm8k_test_514', 'gsm8k_test_515', 'gsm8k_test_516', 'gsm8k_test_517', 'gsm8k_test_518', 'gsm8k_test_519', 'gsm8k_test_520', 'gsm8k_test_521', 'gsm8k_test_522', 'gsm8k_test_523', 'gsm8k_test_524', 'gsm8k_test_525', 'gsm8k_test_526', 'gsm8k_test_527', 'gsm8k_test_528', 'gsm8k_test_529', 'gsm8k_test_530', 'gsm8k_test_531', 'gsm8k_test_532', 'gsm8k_test_533', 'gsm8k_test_534', 'gsm8k_test_535', 'gsm8k_test_536', 'gsm8k_test_537', 'gsm8k_test_538', 'gsm8k_test_539', 'gsm8k_test_540', 'gsm8k_test_541', 'gsm8k_test_542', 'gsm8k_test_543', 'gsm8k_test_544', 'gsm8k_test_545', 'gsm8k_test_546', 'gsm8k_test_547', 'gsm8k_test_548', 'gsm8k_test_549', 'gsm8k_test_550', 'gsm8k_test_551', 'gsm8k_test_552', 'gsm8k_test_553', 'gsm8k_test_554', 'gsm8k_test_555', 'gsm8k_test_556', 'gsm8k_test_557', 'gsm8k_test_558', 'gsm8k_test_559', 'gsm8k_test_560', 'gsm8k_test_561', 'gsm8k_test_562', 'gsm8k_test_563', 'gsm8k_test_564', 'gsm8k_test_565', 'gsm8k_test_566', 'gsm8k_test_567', 'gsm8k_test_568', 'gsm8k_test_569', 'gsm8k_test_570', 'gsm8k_test_571', 'gsm8k_test_572', 'gsm8k_test_573', 'gsm8k_test_574', 'gsm8k_test_575', 'gsm8k_test_576', 'gsm8k_test_577', 'gsm8k_test_578', 'gsm8k_test_579', 'gsm8k_test_580', 'gsm8k_test_581', 'gsm8k_test_582', 'gsm8k_test_583', 'gsm8k_test_584', 'gsm8k_test_585', 'gsm8k_test_586', 'gsm8k_test_587', 'gsm8k_test_588', 'gsm8k_test_589', 'gsm8k_test_590', 'gsm8k_test_591', 'gsm8k_test_592', 'gsm8k_test_593', 'gsm8k_test_594', 'gsm8k_test_595', 'gsm8k_test_596', 'gsm8k_test_597', 'gsm8k_test_598', 'gsm8k_test_599', 'gsm8k_test_600', 'gsm8k_test_601', 'gsm8k_test_602', 'gsm8k_test_603', 'gsm8k_test_604', 'gsm8k_test_605', 'gsm8k_test_606', 'gsm8k_test_607', 'gsm8k_test_608', 'gsm8k_test_609', 'gsm8k_test_610', 'gsm8k_test_611', 'gsm8k_test_612', 'gsm8k_test_613', 'gsm8k_test_614', 'gsm8k_test_615', 'gsm8k_test_616', 'gsm8k_test_617', 'gsm8k_test_618', 'gsm8k_test_619', 'gsm8k_test_620', 'gsm8k_test_621', 'gsm8k_test_622', 'gsm8k_test_623', 'gsm8k_test_624', 'gsm8k_test_625', 'gsm8k_test_626', 'gsm8k_test_627', 'gsm8k_test_628', 'gsm8k_test_629', 'gsm8k_test_630', 'gsm8k_test_631', 'gsm8k_test_632', 'gsm8k_test_633', 'gsm8k_test_634', 'gsm8k_test_635', 'gsm8k_test_636', 'gsm8k_test_637', 'gsm8k_test_638', 'gsm8k_test_639', 'gsm8k_test_640', 'gsm8k_test_641', 'gsm8k_test_642', 'gsm8k_test_643', 'gsm8k_test_644', 'gsm8k_test_645', 'gsm8k_test_646', 'gsm8k_test_647', 'gsm8k_test_648', 'gsm8k_test_649', 'gsm8k_test_650', 'gsm8k_test_651', 'gsm8k_test_652', 'gsm8k_test_653', 'gsm8k_test_654', 'gsm8k_test_655', 'gsm8k_test_656', 'gsm8k_test_657', 'gsm8k_test_658', 'gsm8k_test_659', 'gsm8k_test_660', 'gsm8k_test_661', 'gsm8k_test_662', 'gsm8k_test_663', 'gsm8k_test_664', 'gsm8k_test_665', 'gsm8k_test_666', 'gsm8k_test_667', 'gsm8k_test_668', 'gsm8k_test_669', 'gsm8k_test_670', 'gsm8k_test_671', 'gsm8k_test_672', 'gsm8k_test_673', 'gsm8k_test_674', 'gsm8k_test_675', 'gsm8k_test_676', 'gsm8k_test_677', 'gsm8k_test_678', 'gsm8k_test_679', 'gsm8k_test_680', 'gsm8k_test_681', 'gsm8k_test_682', 'gsm8k_test_683', 'gsm8k_test_684', 'gsm8k_test_685', 'gsm8k_test_686', 'gsm8k_test_687', 'gsm8k_test_688', 'gsm8k_test_689', 'gsm8k_test_690', 'gsm8k_test_691', 'gsm8k_test_692', 'gsm8k_test_693', 'gsm8k_test_694', 'gsm8k_test_695', 'gsm8k_test_696', 'gsm8k_test_697', 'gsm8k_test_698', 'gsm8k_test_699', 'gsm8k_test_700', 'gsm8k_test_701', 'gsm8k_test_702', 'gsm8k_test_703', 'gsm8k_test_704', 'gsm8k_test_705', 'gsm8k_test_706', 'gsm8k_test_707', 'gsm8k_test_708', 'gsm8k_test_709', 'gsm8k_test_710', 'gsm8k_test_711', 'gsm8k_test_712', 'gsm8k_test_713', 'gsm8k_test_714', 'gsm8k_test_715', 'gsm8k_test_716', 'gsm8k_test_717', 'gsm8k_test_718', 'gsm8k_test_719', 'gsm8k_test_720', 'gsm8k_test_721', 'gsm8k_test_722', 'gsm8k_test_723', 'gsm8k_test_724', 'gsm8k_test_725', 'gsm8k_test_726', 'gsm8k_test_727', 'gsm8k_test_728', 'gsm8k_test_729', 'gsm8k_test_730', 'gsm8k_test_731', 'gsm8k_test_732', 'gsm8k_test_733', 'gsm8k_test_734', 'gsm8k_test_735', 'gsm8k_test_736', 'gsm8k_test_737', 'gsm8k_test_738', 'gsm8k_test_739', 'gsm8k_test_740', 'gsm8k_test_741', 'gsm8k_test_742', 'gsm8k_test_743', 'gsm8k_test_744', 'gsm8k_test_745', 'gsm8k_test_746', 'gsm8k_test_747', 'gsm8k_test_748', 'gsm8k_test_749', 'gsm8k_test_750', 'gsm8k_test_751', 'gsm8k_test_752', 'gsm8k_test_753', 'gsm8k_test_754', 'gsm8k_test_755', 'gsm8k_test_756', 'gsm8k_test_757', 'gsm8k_test_758', 'gsm8k_test_759', 'gsm8k_test_760', 'gsm8k_test_761', 'gsm8k_test_762', 'gsm8k_test_763', 'gsm8k_test_764', 'gsm8k_test_765', 'gsm8k_test_766', 'gsm8k_test_767', 'gsm8k_test_768', 'gsm8k_test_769', 'gsm8k_test_770', 'gsm8k_test_771', 'gsm8k_test_772', 'gsm8k_test_773', 'gsm8k_test_774', 'gsm8k_test_775', 'gsm8k_test_776', 'gsm8k_test_777', 'gsm8k_test_778', 'gsm8k_test_779', 'gsm8k_test_780', 'gsm8k_test_781', 'gsm8k_test_782', 'gsm8k_test_783', 'gsm8k_test_784', 'gsm8k_test_785', 'gsm8k_test_786', 'gsm8k_test_787', 'gsm8k_test_788', 'gsm8k_test_789', 'gsm8k_test_790', 'gsm8k_test_791', 'gsm8k_test_792', 'gsm8k_test_793', 'gsm8k_test_794', 'gsm8k_test_795', 'gsm8k_test_796', 'gsm8k_test_797', 'gsm8k_test_798', 'gsm8k_test_799', 'gsm8k_test_800', 'gsm8k_test_801', 'gsm8k_test_802', 'gsm8k_test_803', 'gsm8k_test_804', 'gsm8k_test_805', 'gsm8k_test_806', 'gsm8k_test_807', 'gsm8k_test_808', 'gsm8k_test_809', 'gsm8k_test_810', 'gsm8k_test_811', 'gsm8k_test_812', 'gsm8k_test_813', 'gsm8k_test_814', 'gsm8k_test_815', 'gsm8k_test_816', 'gsm8k_test_817', 'gsm8k_test_818', 'gsm8k_test_819', 'gsm8k_test_820', 'gsm8k_test_821', 'gsm8k_test_822', 'gsm8k_test_823', 'gsm8k_test_824', 'gsm8k_test_825', 'gsm8k_test_826', 'gsm8k_test_827', 'gsm8k_test_828', 'gsm8k_test_829', 'gsm8k_test_830', 'gsm8k_test_831', 'gsm8k_test_832', 'gsm8k_test_833', 'gsm8k_test_834', 'gsm8k_test_835', 'gsm8k_test_836', 'gsm8k_test_837', 'gsm8k_test_838', 'gsm8k_test_839', 'gsm8k_test_840', 'gsm8k_test_841', 'gsm8k_test_842', 'gsm8k_test_843', 'gsm8k_test_844', 'gsm8k_test_845', 'gsm8k_test_846', 'gsm8k_test_847', 'gsm8k_test_848', 'gsm8k_test_849', 'gsm8k_test_850', 'gsm8k_test_851', 'gsm8k_test_852', 'gsm8k_test_853', 'gsm8k_test_854', 'gsm8k_test_855', 'gsm8k_test_856', 'gsm8k_test_857', 'gsm8k_test_858', 'gsm8k_test_859', 'gsm8k_test_860', 'gsm8k_test_861', 'gsm8k_test_862', 'gsm8k_test_863', 'gsm8k_test_864', 'gsm8k_test_865', 'gsm8k_test_866', 'gsm8k_test_867', 'gsm8k_test_868', 'gsm8k_test_869', 'gsm8k_test_870', 'gsm8k_test_871', 'gsm8k_test_872', 'gsm8k_test_873', 'gsm8k_test_874', 'gsm8k_test_875', 'gsm8k_test_876', 'gsm8k_test_877', 'gsm8k_test_878', 'gsm8k_test_879', 'gsm8k_test_880', 'gsm8k_test_881', 'gsm8k_test_882', 'gsm8k_test_883', 'gsm8k_test_884', 'gsm8k_test_885', 'gsm8k_test_886', 'gsm8k_test_887', 'gsm8k_test_888', 'gsm8k_test_889', 'gsm8k_test_890', 'gsm8k_test_891', 'gsm8k_test_892', 'gsm8k_test_893', 'gsm8k_test_894', 'gsm8k_test_895', 'gsm8k_test_896', 'gsm8k_test_897', 'gsm8k_test_898', 'gsm8k_test_899', 'gsm8k_test_900', 'gsm8k_test_901', 'gsm8k_test_902', 'gsm8k_test_903', 'gsm8k_test_904', 'gsm8k_test_905', 'gsm8k_test_906', 'gsm8k_test_907', 'gsm8k_test_908', 'gsm8k_test_909', 'gsm8k_test_910', 'gsm8k_test_911', 'gsm8k_test_912', 'gsm8k_test_913', 'gsm8k_test_914', 'gsm8k_test_915', 'gsm8k_test_916', 'gsm8k_test_917', 'gsm8k_test_918', 'gsm8k_test_919', 'gsm8k_test_920', 'gsm8k_test_921', 'gsm8k_test_922', 'gsm8k_test_923', 'gsm8k_test_924', 'gsm8k_test_925', 'gsm8k_test_926', 'gsm8k_test_927', 'gsm8k_test_928', 'gsm8k_test_929', 'gsm8k_test_930', 'gsm8k_test_931', 'gsm8k_test_932', 'gsm8k_test_933', 'gsm8k_test_934', 'gsm8k_test_935', 'gsm8k_test_936', 'gsm8k_test_937', 'gsm8k_test_938', 'gsm8k_test_939', 'gsm8k_test_940', 'gsm8k_test_941', 'gsm8k_test_942', 'gsm8k_test_943', 'gsm8k_test_944', 'gsm8k_test_945', 'gsm8k_test_946', 'gsm8k_test_947', 'gsm8k_test_948', 'gsm8k_test_949', 'gsm8k_test_950', 'gsm8k_test_951', 'gsm8k_test_952', 'gsm8k_test_953', 'gsm8k_test_954', 'gsm8k_test_955', 'gsm8k_test_956', 'gsm8k_test_957', 'gsm8k_test_958', 'gsm8k_test_959', 'gsm8k_test_960', 'gsm8k_test_961', 'gsm8k_test_962', 'gsm8k_test_963', 'gsm8k_test_964', 'gsm8k_test_965', 'gsm8k_test_966', 'gsm8k_test_967', 'gsm8k_test_968', 'gsm8k_test_969', 'gsm8k_test_970', 'gsm8k_test_971', 'gsm8k_test_972', 'gsm8k_test_973', 'gsm8k_test_974', 'gsm8k_test_975', 'gsm8k_test_976', 'gsm8k_test_977', 'gsm8k_test_978', 'gsm8k_test_979', 'gsm8k_test_980', 'gsm8k_test_981', 'gsm8k_test_982', 'gsm8k_test_983', 'gsm8k_test_984', 'gsm8k_test_985', 'gsm8k_test_986', 'gsm8k_test_987', 'gsm8k_test_988', 'gsm8k_test_989', 'gsm8k_test_990', 'gsm8k_test_991', 'gsm8k_test_992', 'gsm8k_test_993', 'gsm8k_test_994', 'gsm8k_test_995', 'gsm8k_test_996', 'gsm8k_test_997', 'gsm8k_test_998', 'gsm8k_test_999', 'gsm8k_test_1000', 'gsm8k_test_1001', 'gsm8k_test_1002', 'gsm8k_test_1003', 'gsm8k_test_1004', 'gsm8k_test_1005', 'gsm8k_test_1006', 'gsm8k_test_1007', 'gsm8k_test_1008', 'gsm8k_test_1009', 'gsm8k_test_1010', 'gsm8k_test_1011', 'gsm8k_test_1012', 'gsm8k_test_1013', 'gsm8k_test_1014', 'gsm8k_test_1015', 'gsm8k_test_1016', 'gsm8k_test_1017', 'gsm8k_test_1018', 'gsm8k_test_1019', 'gsm8k_test_1020', 'gsm8k_test_1021', 'gsm8k_test_1022', 'gsm8k_test_1023', 'gsm8k_test_1024', 'gsm8k_test_1025', 'gsm8k_test_1026', 'gsm8k_test_1027', 'gsm8k_test_1028', 'gsm8k_test_1029', 'gsm8k_test_1030', 'gsm8k_test_1031', 'gsm8k_test_1032', 'gsm8k_test_1033', 'gsm8k_test_1034', 'gsm8k_test_1035', 'gsm8k_test_1036', 'gsm8k_test_1037', 'gsm8k_test_1038', 'gsm8k_test_1039', 'gsm8k_test_1040', 'gsm8k_test_1041', 'gsm8k_test_1042', 'gsm8k_test_1043', 'gsm8k_test_1044', 'gsm8k_test_1045', 'gsm8k_test_1046', 'gsm8k_test_1047', 'gsm8k_test_1048', 'gsm8k_test_1049', 'gsm8k_test_1050', 'gsm8k_test_1051', 'gsm8k_test_1052', 'gsm8k_test_1053', 'gsm8k_test_1054', 'gsm8k_test_1055', 'gsm8k_test_1056', 'gsm8k_test_1057', 'gsm8k_test_1058', 'gsm8k_test_1059', 'gsm8k_test_1060', 'gsm8k_test_1061', 'gsm8k_test_1062', 'gsm8k_test_1063', 'gsm8k_test_1064', 'gsm8k_test_1065', 'gsm8k_test_1066', 'gsm8k_test_1067', 'gsm8k_test_1068', 'gsm8k_test_1069', 'gsm8k_test_1070', 'gsm8k_test_1071', 'gsm8k_test_1072', 'gsm8k_test_1073', 'gsm8k_test_1074', 'gsm8k_test_1075', 'gsm8k_test_1076', 'gsm8k_test_1077', 'gsm8k_test_1078', 'gsm8k_test_1079', 'gsm8k_test_1080', 'gsm8k_test_1081', 'gsm8k_test_1082', 'gsm8k_test_1083', 'gsm8k_test_1084', 'gsm8k_test_1085', 'gsm8k_test_1086', 'gsm8k_test_1087', 'gsm8k_test_1088', 'gsm8k_test_1089', 'gsm8k_test_1090', 'gsm8k_test_1091', 'gsm8k_test_1092', 'gsm8k_test_1093', 'gsm8k_test_1094', 'gsm8k_test_1095', 'gsm8k_test_1096', 'gsm8k_test_1097', 'gsm8k_test_1098', 'gsm8k_test_1099', 'gsm8k_test_1100', 'gsm8k_test_1101', 'gsm8k_test_1102', 'gsm8k_test_1103', 'gsm8k_test_1104', 'gsm8k_test_1105', 'gsm8k_test_1106', 'gsm8k_test_1107', 'gsm8k_test_1108', 'gsm8k_test_1109', 'gsm8k_test_1110', 'gsm8k_test_1111', 'gsm8k_test_1112', 'gsm8k_test_1113', 'gsm8k_test_1114', 'gsm8k_test_1115', 'gsm8k_test_1116', 'gsm8k_test_1117', 'gsm8k_test_1118', 'gsm8k_test_1119', 'gsm8k_test_1120', 'gsm8k_test_1121', 'gsm8k_test_1122', 'gsm8k_test_1123', 'gsm8k_test_1124', 'gsm8k_test_1125', 'gsm8k_test_1126', 'gsm8k_test_1127', 'gsm8k_test_1128', 'gsm8k_test_1129', 'gsm8k_test_1130', 'gsm8k_test_1131', 'gsm8k_test_1132', 'gsm8k_test_1133', 'gsm8k_test_1134', 'gsm8k_test_1135', 'gsm8k_test_1136', 'gsm8k_test_1137', 'gsm8k_test_1138', 'gsm8k_test_1139', 'gsm8k_test_1140', 'gsm8k_test_1141', 'gsm8k_test_1142', 'gsm8k_test_1143', 'gsm8k_test_1144', 'gsm8k_test_1145', 'gsm8k_test_1146', 'gsm8k_test_1147', 'gsm8k_test_1148', 'gsm8k_test_1149', 'gsm8k_test_1150', 'gsm8k_test_1151', 'gsm8k_test_1152', 'gsm8k_test_1153', 'gsm8k_test_1154', 'gsm8k_test_1155', 'gsm8k_test_1156', 'gsm8k_test_1157', 'gsm8k_test_1158', 'gsm8k_test_1159', 'gsm8k_test_1160', 'gsm8k_test_1161', 'gsm8k_test_1162', 'gsm8k_test_1163', 'gsm8k_test_1164', 'gsm8k_test_1165', 'gsm8k_test_1166', 'gsm8k_test_1167', 'gsm8k_test_1168', 'gsm8k_test_1169', 'gsm8k_test_1170', 'gsm8k_test_1171', 'gsm8k_test_1172', 'gsm8k_test_1173', 'gsm8k_test_1174', 'gsm8k_test_1175', 'gsm8k_test_1176', 'gsm8k_test_1177', 'gsm8k_test_1178', 'gsm8k_test_1179', 'gsm8k_test_1180', 'gsm8k_test_1181', 'gsm8k_test_1182', 'gsm8k_test_1183', 'gsm8k_test_1184', 'gsm8k_test_1185', 'gsm8k_test_1186', 'gsm8k_test_1187', 'gsm8k_test_1188', 'gsm8k_test_1189', 'gsm8k_test_1190', 'gsm8k_test_1191', 'gsm8k_test_1192', 'gsm8k_test_1193', 'gsm8k_test_1194', 'gsm8k_test_1195', 'gsm8k_test_1196', 'gsm8k_test_1197', 'gsm8k_test_1198', 'gsm8k_test_1199', 'gsm8k_test_1200', 'gsm8k_test_1201', 'gsm8k_test_1202', 'gsm8k_test_1203', 'gsm8k_test_1204', 'gsm8k_test_1205', 'gsm8k_test_1206', 'gsm8k_test_1207', 'gsm8k_test_1208', 'gsm8k_test_1209', 'gsm8k_test_1210', 'gsm8k_test_1211', 'gsm8k_test_1212', 'gsm8k_test_1213', 'gsm8k_test_1214', 'gsm8k_test_1215', 'gsm8k_test_1216', 'gsm8k_test_1217', 'gsm8k_test_1218', 'gsm8k_test_1219', 'gsm8k_test_1220', 'gsm8k_test_1221', 'gsm8k_test_1222', 'gsm8k_test_1223', 'gsm8k_test_1224', 'gsm8k_test_1225', 'gsm8k_test_1226', 'gsm8k_test_1227', 'gsm8k_test_1228', 'gsm8k_test_1229', 'gsm8k_test_1230', 'gsm8k_test_1231', 'gsm8k_test_1232', 'gsm8k_test_1233', 'gsm8k_test_1234', 'gsm8k_test_1235', 'gsm8k_test_1236', 'gsm8k_test_1237', 'gsm8k_test_1238', 'gsm8k_test_1239', 'gsm8k_test_1240', 'gsm8k_test_1241', 'gsm8k_test_1242', 'gsm8k_test_1243', 'gsm8k_test_1244', 'gsm8k_test_1245', 'gsm8k_test_1246', 'gsm8k_test_1247', 'gsm8k_test_1248', 'gsm8k_test_1249', 'gsm8k_test_1250', 'gsm8k_test_1251', 'gsm8k_test_1252', 'gsm8k_test_1253', 'gsm8k_test_1254', 'gsm8k_test_1255', 'gsm8k_test_1256', 'gsm8k_test_1257', 'gsm8k_test_1258', 'gsm8k_test_1259', 'gsm8k_test_1260', 'gsm8k_test_1261', 'gsm8k_test_1262', 'gsm8k_test_1263', 'gsm8k_test_1264', 'gsm8k_test_1265', 'gsm8k_test_1266', 'gsm8k_test_1267', 'gsm8k_test_1268', 'gsm8k_test_1269', 'gsm8k_test_1270', 'gsm8k_test_1271', 'gsm8k_test_1272', 'gsm8k_test_1273', 'gsm8k_test_1274', 'gsm8k_test_1275', 'gsm8k_test_1276', 'gsm8k_test_1277', 'gsm8k_test_1278', 'gsm8k_test_1279', 'gsm8k_test_1280', 'gsm8k_test_1281', 'gsm8k_test_1282', 'gsm8k_test_1283', 'gsm8k_test_1284', 'gsm8k_test_1285', 'gsm8k_test_1286', 'gsm8k_test_1287', 'gsm8k_test_1288', 'gsm8k_test_1289', 'gsm8k_test_1290', 'gsm8k_test_1291', 'gsm8k_test_1292', 'gsm8k_test_1293', 'gsm8k_test_1294', 'gsm8k_test_1295', 'gsm8k_test_1296', 'gsm8k_test_1297', 'gsm8k_test_1298', 'gsm8k_test_1299', 'gsm8k_test_1300', 'gsm8k_test_1301', 'gsm8k_test_1302', 'gsm8k_test_1303', 'gsm8k_test_1304', 'gsm8k_test_1305', 'gsm8k_test_1306', 'gsm8k_test_1307', 'gsm8k_test_1308', 'gsm8k_test_1309', 'gsm8k_test_1310', 'gsm8k_test_1311', 'gsm8k_test_1312', 'gsm8k_test_1313', 'gsm8k_test_1314', 'gsm8k_test_1315', 'gsm8k_test_1316', 'gsm8k_test_1317', 'gsm8k_test_1318']}}}, 'scores': [{'arm': 'base', 'metric': 'accuracy', 'value': 0.7035633055344959, 'evaluator_id': 'b9aa01a393125755d46944c706831466ffc955df7a4cad35f711050ce7ef500f', 'seed': 0, 'predictions_ref': 'predictions_base.jsonl'}], 'comparisons': [{'metric': 'accuracy', 'arms': ['grpo_seed0', 'dppo_seed0']}]}

evaluator_validation

{'status': 'unknown', 'issues': ["Comparison provenance is incomplete; record every declared arm's score and evaluator."], 'comparisons': [{'metric': 'accuracy', 'status': 'unknown', 'arms': ['base', 'dppo_seed0', 'grpo_seed0'], 'evaluator_ids': ['b9aa01a393125755d46944c706831466ffc955df7a4cad35f711050ce7ef500f'], 'missing_arms': ['dppo_seed0', 'grpo_seed0']}]}

feasibility

{'status': 'unknown', 'problems': ['Re-projection required: base_model changed or was not recorded', 'Re-projection required: lora_rank changed or was not recorded', 'Re-projection required: dataset changed or was not recorded', 'Re-projection required: steps changed or was not recorded', 'Re-projection required: batch_size changed or was not recorded', 'Re-projection required: max_seq_len changed or was not recorded', 'Re-projection required: max_tokens changed or was not recorded', 'Re-projection required: arms changed or was not recorded', 'Re-projection required: n_examples changed or was not recorded', 'Re-projection required: n_eval changed or was not recorded', 'Representative training timing not recorded', 'Representative evaluation timing not recorded', 'Unknown timing phase: training_pilot', 'Unknown timing phase: evaluation_base', 'Positive recovery reserve not recorded'], 'wall_seconds': None, 'gpu_hours': None, 'known_gpu_hours': 0, 'phases': {'setup': {'wall_seconds': 89.47155904769896, 'planned_units': 3, 'planned_samples': 3.0, 'gpu_hours': None}}, 'generation_counts': None, 'generation_applicability': {'check': 'generation_audit', 'status': 'not_applicable', 'enforce': False, 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'features': {'generated_training_data': False}}, 'record': {'source': 'measured', 'primary_endpoint': 'gsm8k_test_greedy_exact_match_accuracy@final_checkpoint (1319 items)', 'config': {'steps_per_arm': 50, 'arms': 2, 'groups_per_batch': 32, 'group_size': 8, 'eval_items': 1319}, 'effective_config': {'base_model': 'Qwen/Qwen3.5-2B', 'lora_rank': 32, 'lora_init_seed': None, 'data_seed': 0, 'optimizer': {'type': 'adamw', 'learning_rate': 2e-05, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'renderer': 'qwen3_5_disable_thinking', 'env': 'tinker_cookbook 0.5.9 math_rl Gsm8kDataset/MathEnv, standard fewshot prefix, format_coef 0.1', 'groups_per_batch': 32, 'group_size': 8, 'temperature': 1.0, 'max_tokens': 512, 'advantages': 'group-mean centered (cookbook compute_advantages)', 'substeps_per_batch': 2, 'minibatch_split': 'groups 0-15 / 16-31', 'planned_steps': 50, 'executed_pilot_steps': 5, 'full_training_executed': False, 'grpo': {'eps': 0.2}, 'dppo': {'variant': 'Binary-TV', 'delta': 0.15, 'clip_ratio_c': 10.0}, 'loss_normalization': 'token sum', 'endpoint': 'shared Spindle (TINKER_BASE_URL)', 'init_state_path': 'tinker://21e32976f18f971463410adff4e364e4/weights/rx19badd842c_init', 'eval': {'temperature': 0.0, 'max_tokens': 512, 'stop': [248046], 'num_samples': 1}}, 'measurements': {'setup': {'seconds': 59.64770603179931, 'units': 2, 'planned_units': 3, 'samples': 2, 'gpu_count': None, 'allocation_source': None}, 'training_pilot': {'seconds': 299.02492904663086, 'units': 5, 'planned_units': 100, 'samples': 5, 'gpu_count': None, 'allocation_source': None}, 'evaluation_base': {'seconds': 201.11393427848816, 'units': 1319, 'planned_units': 3957, 'samples': 1, 'gpu_count': None, 'allocation_source': None}}, 'generation': None, 'recovery': {'seconds': 0, 'gpu_count': None, 'allocation_source': None}}, 'recorded_at': 1793741452.0510879}

feasibility_required

True

generation_audit_validation

{'status': 'not_applicable', 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'applicability': {'check': 'generation_audit', 'status': 'not_applicable', 'enforce': False, 'reasons': ['spec.generation_audit opts out and no feature it covers is true'], 'features': {'generated_training_data': False}}}

initialization_provenance

{'version': 1, 'arms': [{'arm': 'dppo_pilot', 'base_checkpoint': 'Qwen/Qwen3.5-2B', 'source': 'server_response', 'checkpoint_uri': 'tinker://21e32976f18f971463410adff4e364e4/weights/rx19badd842c_init', 'loaded_checkpoint_uri': 'tinker://21e32976f18f971463410adff4e364e4/weights/rx19badd842c_init', 'load_succeeded': True, 'checkpoint_immutable': True, 'checkpoint_scope': 'full_initialization', 'before_first_update': True, 'evidence_ref': 'pilot-restore-init.json', 'optimizer': {'source': 'server_response', 'type': 'adamw', 'declared_step_params': {'learning_rate': 2e-05, 'beta1': 0.9, 'beta2': 0.95, 'eps': 1e-08, 'weight_decay': 0.0, 'grad_clip_norm': 0.0}, 'betas_eps_actual': None}, 'initial_output': {'available': True, 'batch_sha256': '5c545b6102e24b6f68b2a5e593ad2f7a81d85c4c170b97533ac4bc0f494d772b', 'output_sha256': '10572d61d17d3ec76f755aa2123b85033e381e883af6983427af9fb01edab837', 'evidence_ref': 'pilot-restore-init.json', 'before_first_update': True}, 'note': 'Paired grpo_seed0/dppo_seed0 clients were never created because the activation gate failed; cross-arm restoration comparison was not performed.'}]}

launch

{'experiment_id': 'exp_e6b0965af3', 'context_status': 'snapshotted', 'superseded': False, 'mode': 'ordinary', 'purpose': 'Execute the selected saved experiment version.'}

launch_instructions

{'name': 'launch-instructions-5ef650264f18.md', 'sha256': '5ef650264f1800e7db58f515dfac944bbb6bfe48af3e3d55eedbad943b614f4a', 'url': 'https://ta-01m6fva5db548f5skm214fepcr-8000-oah9l7x00jpo7186eb6cix5v0.w.modal.host/api/runs/run_19badd842c/evidence/5ef650264f1800e7db58f515dfac944bbb6bfe48af3e3d55eedbad943b614f4a'}

reported_outcome_reason

incomplete_execution

reported_verdict

inconclusive

restoration_validation

{'status': 'unverified', 'required': True, 'checks': [{'check': 'grpo_seed0', 'status': 'unverified', 'reason': 'grpo_seed0: missing or duplicate arm evidence.'}, {'check': 'dppo_seed0', 'status': 'unverified', 'reason': 'dppo_seed0: missing or duplicate arm evidence.'}], 'reason': 'grpo_seed0: missing or duplicate arm evidence.', 'applicability': 'required'}