import json, hashlib, os, importlib.metadata as md, calendar, time H = lambda f: hashlib.sha256(open(f, "rb").read()).hexdigest() URL = { "scored-observations.jsonl": "https://modal.devinenterprise.com/attachments/63d47296-fbd6-4adc-bcf3-9e6945f9522c/scored-observations.jsonl", "predictions-confirmation-treatment.jsonl": "https://modal.devinenterprise.com/attachments/f23f0d58-d1d2-4cae-baa9-15d0b8a4cbd5/predictions-confirmation-treatment.jsonl", "predictions-confirmation-baseline.jsonl": "https://modal.devinenterprise.com/attachments/cf0a33fc-02f9-4131-8a45-6ab2b6f1175b/predictions-confirmation-baseline.jsonl", "predictions-confirmation-untrained_base.jsonl": "https://modal.devinenterprise.com/attachments/9d1731c9-5239-4fb9-905b-c8e7f5cd92e9/predictions-confirmation-untrained_base.jsonl", "predictions-selection.jsonl": "https://modal.devinenterprise.com/attachments/c3ae7b95-1e72-4ba1-9196-cd45ce8b0157/predictions-selection.jsonl", "training-metrics.jsonl": "https://modal.devinenterprise.com/attachments/6f5ffdfa-a03a-4214-ac6b-e4d8fc9cad2d/training-metrics.jsonl", "checkpoints.jsonl": "https://modal.devinenterprise.com/attachments/fd06c2f2-af6b-4a81-8630-6cc36fde1111/checkpoints.jsonl", "client-events.jsonl": "https://modal.devinenterprise.com/attachments/621b0fc8-a827-4568-b902-18db4bf167a1/client-events.jsonl", "decision-result.json": "https://modal.devinenterprise.com/attachments/285277c8-9f42-43ed-aa3d-b3262e8c7d2e/decision-result.json", "restoration-validation.json": "https://modal.devinenterprise.com/attachments/2e8f6ce0-9ebe-48fa-a564-085cfadf21c4/restoration-validation.json", "diag-init.json": "https://modal.devinenterprise.com/attachments/f5bfc7b7-5b1f-4569-a7fd-a4660d96ccf6/diag-init.json", "launch-instructions.md": "https://modal.devinenterprise.com/attachments/d24196f4-6610-4d29-9111-9ecafc203130/launch-instructions.md"} KIND = {"launch-instructions.md": "instructions", "scored-observations.jsonl": "metrics", "training-metrics.jsonl": "metrics", "checkpoints.jsonl": "metrics", "client-events.jsonl": "metrics", "decision-result.json": "metrics", "restoration-validation.json": "metrics", "diag-init.json": "metrics"} files = [{"name": n, "kind": KIND.get(n, "predictions"), "sha256": H("ev/" + n), "source_url": u} for n, u in URL.items()] for n in ("selection-frozen.json", "probe-results.json", "feasibility-measurements.json"): files.append({"name": n, "kind": "metrics", "sha256": H("ev/" + n), "content": open("ev/" + n).read()}) for s in ("common.py", "run_arm.py", "launch.sh", "probe.py", "fix.py", "diag_init.py", "evalgsm.py", "sel_all.sh", "sel_ext.sh", "analyze.py", "conf.sh", "build_provenance.py", "feas.py", "build_final.py"): files.append({"name": s, "kind": "script", "sha256": H(s), "content": open(s).read()}) sel = json.load(open("selection_frozen.json")); dec = json.load(open("decision_result.json")) eff = {"base_model": "Qwen/Qwen2.5-3B", "renderer_name": "role_colon", "env": "gsm8k", "loss_fn": "importance_sampling", "group_size": 8, "groups_per_batch": 8, "batch_rollouts": 64, "max_tokens": 512, "max_steps": 40, "seed": 0, "optimizer": "adamw (server get_info)", "lr_grid": {"1": sel["ranks"]["1"]["grid"], "32": sel["ranks"]["32"]["grid"]}, "selected_lr": {"1": sel["ranks"]["1"]["selected_lr"], "32": sel["ranks"]["32"]["selected_lr"]}, "init_checkpoints": {"1": "tinker://858fb34c3b2a340a600975ef5aaa34c2/weights/run_b89b93886b-init-r1", "32": "tinker://879b6d3bac21f78eff5ba092b3b3b86b/weights/run_b89b93886b-init-r32"}, "eval": {"selection_items": "gsm8k/test/0..199", "confirmation_items": "gsm8k/test/200..1318", "n_confirmation": 1119, "decoding": "greedy temperature 0, max_tokens 512", "metric": "numeric exact match of boxed answer"}, "endpoint": "shared Spindle via TINKER_BASE_URL (Tinker 0.37 default_headers_overrides proxy headers)"} open("ev/effective-config.json", "w").write(json.dumps(eff, indent=1)) files.append({"name": "effective-config.json", "kind": "effective_config", "sha256": H("ev/effective-config.json"), "content": json.dumps(eff, indent=1)}) vers = {p: md.version(p) for p in ("tinker", "tinker-cookbook", "datasets", "transformers", "math-verify")} rev = json.dumps({"packages": vers, "research_exchange_repo_head": "77745cd9581c58c707f442c1ff8f8de602413988 (read-only; decision/restoration validators)", "repo_changes": "none"}, indent=1) files.append({"name": "code-revision.json", "kind": "code_revision", "content": rev}) # checkpoints ck = [] for l in open("ev/checkpoints.jsonl"): r = json.loads(l) if r["name"] == "final" and "pilot" not in r["arm_dir"]: a = r["arm_dir"].replace("r1_", "rank1_").replace("r32_", "rank32_") ck.append({"arm": a, "step": r["batch"], "uri": r["state_path"]}); ck.append({"arm": a + ":sampler", "step": r["batch"], "uri": r["sampler_path"]}) # arms metrics fm = json.load(open("feasibility_measurements.json")) arms = {"pilot_rank1_lr3e-4": {"step0_mean_reward": 0.51875, "step0_frac_mixed": 1.0, "executed_lora_rank": 1}, "pilot_rank32_lr3e-4": {"step0_mean_reward": 0.5125, "step0_frac_mixed": 1.0, "executed_lora_rank": 32}, "base": {"selection_accuracy": sel["base_selection_accuracy"], "confirmation_accuracy": dec["base_acc"]}} for R in ("1", "32"): for lr, a in sel["ranks"][R]["selection_accuracy"].items(): m = [json.loads(l) for l in open(f"runs/r{R}_lr{lr}/metrics.jsonl")] m = [x for x in m if "env/all/correct" in x] arms[f"rank{R}_lr{lr}"] = {"selection_accuracy": a, "lora_rank": int(R), "learning_rate": float(lr), "train_steps": len(m), "train_correct_step0": m[0]["env/all/correct"], "train_correct_last": m[-1]["env/all/correct"], "train_wall_seconds": fm[f"r{R}_lr{lr}"]["seconds"]} arms["treatment"] = {"confirmation_accuracy": dec["treatment_acc"], "lora_rank": 1, "learning_rate": 1e-4} arms["baseline"] = {"confirmation_accuracy": dec["baseline_acc"], "lora_rank": 32, "learning_rate": 1e-4} arms["untrained_base"] = {"confirmation_accuracy": dec["base_acc"]} crit = dec["assess_expanded_ids"]["criteria"][0] prov = json.load(open("restoration_validation.json")) start = calendar.timegm((2026, 11, 4, 16, 49, 0)); pilot0 = int(open("runs/pilot_r1_start").read()) feas = {"config": {"base_model": "Qwen/Qwen2.5-3B", "arms": ["rank1_lr3e-4", "rank32_lr3e-4"], "steps": 3, "batch_size": 64, "max_tokens": 512, "n_eval": 200}, "effective_config": {"base_model": "Qwen/Qwen2.5-3B", "arms": [k for k in arms if k.startswith("rank")], "steps": 40, "batch_size": 64, "group_size": 8, "groups_per_batch": 8, "max_tokens": 512, "n_eval": {"selection": 200, "confirmation": 1119}}, "primary_endpoint": "gsm8k_test_confirmation_exact_match_accuracy", "source": "feasibility-measurements.json; runs/pilot_*; evals/sel_base.log", "measurements": { "setup": {"seconds": pilot0 - start, "units": 1, "planned_units": 1, "samples": 0, "gpu_count": None, "allocation_source": None}, "training": {"seconds": fm["pilot_r1_lr3e-4"]["seconds"] + fm["pilot_r32_lr3e-4"]["seconds"], "units": 6, "planned_units": 6 + 8 * 40, "samples": 384, "gpu_count": None, "allocation_source": None}, "evaluation": {"seconds": 10.34, "units": 200, "planned_units": 8 * 200 + 200 + 3 * 1119, "samples": 200, "gpu_count": None, "allocation_source": None}}, "generation": None, "recovery": {"seconds": 1800, "gpu_count": None, "allocation_source": None}} out = {"metrics": {"arms": arms, "effect": dec["effect"], "ci_lower": crit["lower"], "ci_upper": crit["upper"], "n_confirmation_items": 1119}, "effect": dec["effect"], "margin": -0.09, "ci": [crit["lower"], crit["upper"]], "seeds": 1, "grid_edge": False, "evaluation": dec["evaluation"], "decision": {"version": 1, "observations_file": {"name": "scored-observations.jsonl", "sha256": H("ev/scored-observations.jsonl")}, "selection": sel}, "evidence": {"version": 1, "files": files, "checkpoints": ck, "per_arm_metrics": {k: v for k, v in arms.items() if k in ("treatment", "baseline", "untrained_base")}}, "initialization_provenance": prov["initialization_provenance"], "feasibility": feas} json.dump(out, open("final_parts.json", "w"), indent=1) print(len(files), "files", sum(len(f.get("content", "")) for f in files), "inline bytes", len(ck), "ckpts", crit, feas["measurements"]["setup"])