import json, hashlib, sys from pathlib import Path R = Path(__file__).resolve().parent J = lambda p: json.loads((R / p).read_text()) sha = lambda p: hashlib.sha256((R / p).read_bytes()).hexdigest() urls = json.loads(Path(sys.argv[1]).read_text()) if len(sys.argv) > 1 else {} res = J("full/result.json"); dec = res["decision"]; hoe = J("paired-hoeffding.json") ev = J("evaluation-full.json"); base = J("evaluation-base.json") scores = {s["arm"]: s for s in ev["scores"]} def arm_metrics(name): steps = [J(f"full/{name}/step-{n:03d}.json") for n in range(1, 41)] s = scores[name] m = dict(accuracy=s["accuracy"], correct=s["correct"], n_eval=s["items"], steps=len(steps), mean_train_reward=sum(x["reward_mean"] for x in steps) / 40, frac_nonzero_advantage=sum(x["frac_nonzero_advantage"] for x in steps) / 40, mean_step_seconds=sum(x["timings"]["step_seconds"] for x in steps) / 40, executed_group_size=float(steps[0]["executed_group_size"][0])) if name == "gs1_kernel": m.update(bandwidth=1.0, min_step_kernel_neff_p10=min(x["neff_p10"] for x in steps)) return m arms = {a: arm_metrics(a) for a in ("grpo_gs8", "gs1_kernel")} arms["base"] = dict(accuracy=base["accuracy"], correct=base["correct"], n_eval=base["items"]) effect = arms["gs1_kernel"]["accuracy"] - arms["grpo_gs8"]["accuracy"] print(json.dumps(dict(arms=arms, effect=effect, hoeffding=hoe, decision=dec), indent=1, default=str)[:3000]) evaluation = dict(ev); evaluation["scores"] = ev["scores"] + [dict(arm="base", metric="accuracy", value=base["accuracy"], evaluator_id=base["evaluator_id"], seed=0, predictions_ref="predictions-base.jsonl")] for s in evaluation["scores"]: if s["arm"] != "base": s["predictions_ref"] = f"predictions-{s['arm']}.jsonl" inline = [("launch-instructions.md","instructions"),("full_run.py","script"),("rx_gs1.py","script"),("gs1_reference.py","script"), ("eval.py","script"),("pilot.py","script"),("run_preflight.py","script"),("remote_stages.py","script"),("offline_tests.py","script"), ("seed_probe.py","script"),("assemble.py","script"),("effective-spec.json","effective_config"),("pins.json","code_revision"), ("decision-full.json","metrics"),("paired-hoeffding.json","metrics"),("preflight.json","metrics"),("validator-effective.json","metrics"), ("validator-original.json","metrics"),("restoration-validation.json","metrics"),("initialization_provenance_full.json","metrics"), ("initialization_provenance.json","metrics"),("probe.json","metrics"),("offline-advantage-tests.json","metrics"),("seed-rejection.log","metrics"), ("activation-calibration.json","metrics"),("activation-pilot-gate-bw0.5.json","metrics"),("activation-pilot-gate-bw1.0.json","metrics"), ("measurements.json","metrics"),("evaluation-base.json","metrics"),("full/timings.json","metrics")] files = [dict(name=n.replace("/", "__"), kind=k, content=(R / n).read_text(), sha256=sha(n)) for n, k in inline] for name, (kind, path) in {"scored-observations.jsonl": ("metrics", "scored-observations.jsonl"), "predictions-grpo_gs8.jsonl": ("predictions", "full/grpo_gs8/eval.jsonl"), "predictions-gs1_kernel.jsonl": ("predictions", "full/gs1_kernel/eval.jsonl"), "predictions-base.jsonl": ("predictions", "base/eval.jsonl"), "evaluator.json": ("metrics", "evaluator.json"), "evaluation-full.json": ("metrics", "evaluation-full.json"), "evaluation-items.jsonl": ("metrics", "evaluation-items.jsonl"), "activation-pilot-treatment-bw0.5.json": ("metrics", "activation-pilot-treatment-bw0.5.json"), "activation-pilot-treatment-bw1.0.json": ("metrics", "activation-pilot-treatment-bw1.0.json"), "activation-pilot-baseline.json": ("metrics", "activation-pilot-baseline.json"), "fixed-batch.json": ("metrics", "fixed-batch.json")}.items(): if path in urls: files.append(dict(name=name, kind=kind, source_url=urls[path], sha256=sha(path))) ck = [] for a in ("grpo_gs8", "gs1_kernel"): f = J(f"full/{a}/state.json")["final"] ck += [dict(arm=a, step=40, uri=f["state"]["path"]), dict(arm=a, step=40, uri=f["sampling"]["path"])] ck.append(dict(arm="init", step=0, uri=J("init-save-response.json")["path"])) pre = J("preflight.json") out = dict(status="succeeded", metrics=dict(base_gsm8k_test_accuracy=base["accuracy"], effect_gs1_minus_grpo=effect, hoeffding_lower=hoe["lower"], hoeffding_upper=hoe["upper"], driver_wall_seconds=res["wall_seconds"], arms=arms), completed_arms=["grpo_gs8", "gs1_kernel"], effect=effect, margin=-0.03, ci=[hoe["lower"], hoe["upper"]], seeds=1, grid_edge=False, deviations=[], original_hypothesis_tested=True, decision=dict(version=1, observations_file=dict(name="scored-observations.jsonl", sha256=sha("scored-observations.jsonl"))), evaluation=evaluation, correctness_preflight=pre, initialization_provenance=J("initialization_provenance_full.json"), evidence=dict(version=1, files=files, checkpoints=ck, per_arm_metrics=arms)) (R / "structured-draft.json").write_text(json.dumps(out, default=str)) print("files", len(files), "bytes", len(json.dumps(out, default=str)))