import json, hashlib, os
O = "ev/out"
URLS = {l.split("/")[-1]: l for l in """https://modal.devinenterprise.com/attachments/08970d7d-305f-4121-bd54-a8da212074c1/fixed-batch-and-initial-logprobs.json
https://modal.devinenterprise.com/attachments/db894b37-50d6-4a07-bfab-c8bf58023019/init-provenance-bundle.json
https://modal.devinenterprise.com/attachments/a0f34cb7-0e5e-405b-878a-360648543272/logs-excerpts.json
https://modal.devinenterprise.com/attachments/4da3f141-4728-4f03-9a31-cbe9626730f7/pred_confirmation_base.jsonl
https://modal.devinenterprise.com/attachments/c054c3d6-2739-419a-8322-4e20fb023c6a/pred_confirmation_rank1_lr3e-5.jsonl
https://modal.devinenterprise.com/attachments/c2c4f20a-f467-478f-aa0f-74b00bba0acf/pred_confirmation_rank32_lr3e-4.jsonl
https://modal.devinenterprise.com/attachments/9bac23ce-2f80-4a99-9999-ab4ab08ea5f5/pred_selection_base.jsonl
https://modal.devinenterprise.com/attachments/7fec3979-8950-4b49-b49a-6fa51ea796ca/pred_selection_rank1_lr1e-3.jsonl
https://modal.devinenterprise.com/attachments/1b11724b-4205-41de-89b7-aa64faa3c020/pred_selection_rank1_lr1e-4.jsonl
https://modal.devinenterprise.com/attachments/9605268e-0fe7-4951-b42b-456528be6644/pred_selection_rank1_lr3e-4.jsonl
https://modal.devinenterprise.com/attachments/6aa09ef3-6812-4d40-b4a9-a6b84f035712/pred_selection_rank1_lr3e-5.jsonl
https://modal.devinenterprise.com/attachments/a1ad3b66-c659-4ceb-b54f-e50b2fc9c5d9/pred_selection_rank32_lr1e-3.jsonl
https://modal.devinenterprise.com/attachments/c286ebff-d996-4411-a95a-e78b8926d838/pred_selection_rank32_lr1e-4.jsonl
https://modal.devinenterprise.com/attachments/bd8b3321-3310-4ec5-8186-86680f6d39e1/pred_selection_rank32_lr3e-4.jsonl
https://modal.devinenterprise.com/attachments/8700168d-857b-408f-9f9b-0372a8ffc85c/scored-observations.jsonl
https://modal.devinenterprise.com/attachments/f891887b-2589-4be2-a55c-a6040d38661f/training-metrics.json""".split()}
KIND = {"common.py": "script", "probe.py": "script", "train_arm.py": "script", "eval_gsm8k.py": "script", "select_lr.py": "script",
        "build_report.py": "script", "auto_sel.sh": "script", "assemble.py": "script", "final.py": "script", "code-revision.json": "code_revision",
        "eval-scores-and-selection.json": "metrics", "probe-results.json": "metrics", "init-provenance-bundle.json": "metrics",
        "fixed-batch-and-initial-logprobs.json": "metrics", "logs-excerpts.json": "metrics", "training-metrics.json": "metrics",
        "scored-observations.jsonl": "metrics"}
import shutil; shutil.copy("final.py", f"{O}/final.py")
rp = json.load(open("ev/report_parts.json")); prim = rp["primary"]; sel = json.load(open("ev/selection.json"))
eff = {"base_model": "Qwen/Qwen2.5-3B", "base_revision": "3aab1f1954e9cc14eb9509a215f9e5ca08227a9b", "recipe": "tinker_cookbook.recipes.math_rl.train (cli_main, stock) @f79fdb2b",
       "recipe_args": {"env": "gsm8k", "group_size": 8, "groups_per_batch": 8, "max_tokens": 512, "max_steps": 40, "seed": 0, "loss_fn": "importance_sampling",
                       "renderer_name": "role_colon", "eval_every": 0, "save_every": 20, "temperature": 1.0, "num_substeps": 1, "kl_penalty_coef": 0.0},
       "optimizer": "recipe AdamParams(lr, beta1=0.9, beta2=0.95, eps=1e-8), constant LR",
       "arms": {"rank1": ["3e-5 (grid-edge extension)", "1e-4", "3e-4", "1e-3"], "rank32": ["1e-4", "3e-4", "1e-3"]},
       "notes": ["cookbook model_info has no Qwen2.5-3B entry, so renderer_name=role_colon (cookbook base-model renderer) was passed explicitly",
                 "cookbook CLI default loss_fn is now stable_reinforce; loss_fn=importance_sampling passed to match the spec's IS loss",
                 "eval_every=0 disables the recipe's in-training test-set eval (logging only; training unchanged)",
                 "rank32_lr1e-4 and rank32_lr3e-4 died with 410 model_lost at steps 38/36 and were resumed by the stock recipe from their step-20 state (with optimizer); steps 20-39 re-ran",
                 "wrapper train_arm.py only injects Modal proxy headers and records read-only provenance"],
       "n_eval": {"selection": 200, "confirmation": 1119}}
files = []
for f in sorted(os.listdir(O)):
    b = open(f"{O}/{f}", "rb").read(); h = hashlib.sha256(b).hexdigest()
    kind = KIND.get(f, "predictions" if f.startswith("pred_") else "metrics")
    e = {"name": f, "kind": kind, "sha256": h}
    if f in URLS: e["source_url"] = URLS[f]
    else: e["content"] = b.decode()
    files.append(e)
files.append({"name": "effective-config.json", "kind": "effective_config", "content": json.dumps(eff, indent=1)})
paths = dict(l.split() for l in open("logs/final_paths.txt"))
arms_m = rp["arms"]
for a, v in arms_m.items():
    if a in paths: v["final_sampler_checkpoint_step"] = 40
arms_m["base"]["confirmation_accuracy"] = prim["base_acc"]
arms_m[prim["roles"]["treatment"]]["confirmation_accuracy"] = prim["treatment_acc"]
arms_m[prim["roles"]["baseline"]]["confirmation_accuracy"] = prim["baseline_acc"]
metrics_arms = {a: {k: v for k, v in m.items() if isinstance(v, (int, float)) and not isinstance(v, bool)} for a, m in arms_m.items()}
metrics_arms["treatment"] = {"confirmation_accuracy": prim["treatment_acc"]}; metrics_arms["baseline"] = {"confirmation_accuracy": prim["baseline_acc"]}
prov = rp["initialization_provenance"]
for a in prov["arms"]:
    for k in ("initialization", "initial_output"): a[k]["evidence_ref"] = "init-provenance-bundle.json"
    for o in a["optimizer_checks"]: o["evidence_ref"] = "init-provenance-bundle.json"
    for r in a.get("restores", []): r["evidence_ref"] = "init-provenance-bundle.json"
obs_sha = hashlib.sha256(open(f"{O}/scored-observations.jsonl", "rb").read()).hexdigest()
iv = prim["interval"]
out = {
 "status": "succeeded",
 "summary": "Stock tinker-cookbook math_rl GRPO-style (importance-sampling) runs on Qwen/Qwen2.5-3B on shared Spindle: 7 LoRA arms over 40 steps (rank 1 at LR 3e-5/1e-4/3e-4/1e-3, rank 32 at 1e-4/3e-4/1e-3), with LR picked on GSM8K test[0:200] and scored once on test[200:1319]. Tuned rank-1 (LR 3e-5) reached 0.8239 and tuned rank-32 (LR 3e-4) reached 0.8105 vs base 0.7346, a paired difference of +0.0134 with Hoeffding 95% CI [-0.068, +0.095], above the -0.09 margin; however the rank-1 pick sits on the edge of its extended grid (a three-way tie at 0.82), so under the spec's rule the result is inconclusive.",
 "metrics": {"effect_rank1_minus_rank32": prim["treatment_acc"] - prim["baseline_acc"], "ci_lower": iv["lower"], "ci_upper": iv["upper"],
             "treatment_confirmation_accuracy": prim["treatment_acc"], "baseline_confirmation_accuracy": prim["baseline_acc"],
             "base_confirmation_accuracy": prim["base_acc"], "base_selection_accuracy": arms_m["base"]["selection_accuracy"],
             "treatment_minus_base": prim["treatment_acc"] - prim["base_acc"], "baseline_minus_base": prim["baseline_acc"] - prim["base_acc"],
             "n_confirmation_items": prim["n"], "selected_lr_rank1": 3e-5, "selected_lr_rank32": 3e-4, "arms": metrics_arms},
 "completed_arms": ["rank1_lr1e-4", "rank1_lr3e-4", "rank1_lr1e-3", "rank32_lr1e-4", "rank32_lr3e-4", "rank32_lr1e-3", "rank1_lr3e-5"],
 "verdict": "refuted", "confidence": "weak", "outcome_reason": "insufficient_evidence",
 "effect": prim["treatment_acc"] - prim["baseline_acc"], "margin": -0.09, "ci": [iv["lower"], iv["upper"]], "seeds": 1, "grid_edge": True,
 "reasoning": "The statistical criterion passes: the tuned rank-1 minus tuned rank-32 paired Hoeffding 95% CI over 1119 confirmation items is [-0.068, +0.095], so its lower bound clears the -0.09 margin, and both selected arms beat base by at least 0.05 (+0.089 and +0.076). The spec's success criterion also says a selected LR on a grid edge that cannot be extended is inconclusive. Rank-1's original pick, 1e-4 (a tie with 3e-4 at 0.82), was on the edge; the one allowed extension, 3e-5, also scored 0.82, so the lower-LR tie rule picked 3e-5, again an edge. Because the criterion is not met as written, this is recorded as not supported (weak) rather than supported. Confidence is further limited by one seed, by initialization/optimizer-config provenance that shared Spindle leaves unverified (no adapter download; get_info exposes only optimizer type adamw), and by the spec's string-range item IDs, which make the shared decision validator reject the adaptive-selection plan (with explicit ID lists the same observations assess as passed). LR 1e-3 collapsed both ranks to 0.0.",
 "deviations": [], "original_hypothesis_tested": True,
 "correctness_preflight": {"required": False, "paths": []},
 "activation_calibration": {"values": {"executed_lora_rank_rank1_arm": "1", "executed_lora_rank_rank32_arm": "32",
     "frac_groups_nonzero_advantage_step0": 0.875, "mean_train_reward_step0": 0.6953125,
     "frac_groups_nonzero_advantage_step0_rank1": 0.875, "frac_groups_nonzero_advantage_step0_rank32": 1.0,
     "mean_train_reward_step0_rank1": 0.75, "mean_train_reward_step0_rank32": 0.640625},
     "note": "3-step pilots rank1_lr3e-4 and rank32_lr3e-4 on the exact full-run stock recipe (importance_sampling, group 8 x 8, max_tokens 512, seed 0, role_colon). Rank comes from server get_info. frac_groups_nonzero_advantage_step0 is the minimum of the recipe's env/all/by_group/frac_mixed at step 0 over both pilots. mean_train_reward_step0 is the mean of env/all/correct at step 0 over both pilots. All six full arms also had step-0 frac_mixed=1.0 and step-0 correct 0.64-0.75."},
 "initialization_provenance": prov,
 "evaluation": rp["evaluation"],
 "decision": {"version": 1, "observations_file": {"name": "scored-observations.jsonl", "sha256": obs_sha},
              "selection": {**sel, "search_item_ids": "gsm8k/test/0..199", "confirmation_item_ids": "gsm8k/test/200..1318",
                            "treatment_arm_checkpoint": "rank1_lr3e-5", "baseline_arm_checkpoint": "rank32_lr3e-4"}},
 "evidence": {"version": 1, "files": files,
              "checkpoints": [{"arm": a, "step": 40, "uri": p} for a, p in paths.items()],
              "per_arm_metrics": metrics_arms},
 "feasibility": {"config": {"base_model": "Qwen/Qwen2.5-3B", "arms": ["rank1_lr3e-4", "rank32_lr3e-4"], "steps": 3, "group_size": 8, "groups_per_batch": 8, "max_tokens": 512, "n_eval": 200},
   "effective_config": {"base_model": "Qwen/Qwen2.5-3B", "arms": ["rank1_lr3e-5", "rank1_lr1e-4", "rank1_lr3e-4", "rank1_lr1e-3", "rank32_lr1e-4", "rank32_lr3e-4", "rank32_lr1e-3"],
                        "steps": 40, "batch_size": 64, "groups_per_batch": 8, "group_size": 8, "max_tokens": 512, "loss_fn": "importance_sampling", "renderer": "role_colon",
                        "n_eval": {"selection": 1600, "confirmation": 3357}},
   "primary_endpoint": "gsm8k_test_confirmation_exact_match_accuracy",
   "source": "training-metrics.json, eval-scores-and-selection.json, probe-results.json, logs-excerpts.json",
   "measurements": {"setup": {"seconds": 31.1, "units": 1, "planned_units": 1, "samples": 0, "gpu_count": None, "allocation_source": None},
                    "training": {"seconds": 2900, "units": 320, "planned_units": 320, "samples": 20480, "gpu_count": None, "allocation_source": None},
                    "evaluation": {"seconds": 391.4, "units": 4957, "planned_units": 4957, "samples": 4957, "gpu_count": None, "allocation_source": None}},
   "generation": None, "recovery": {"seconds": 1200, "gpu_count": None, "allocation_source": None}},
}
json.dump(out, open("ev/final.json", "w"), indent=1)
print(len(json.dumps(out)), len(files), sum(len(f.get("content", "")) for f in files))