--- a/code/losses.py +++ b/code/losses.py @@ -36,13 +36,24 @@ n = int(m.sum()) if n == 0: return {} - lp, olp, a = logp[m].double(), old_lp[m].double(), adv[m].double() + # Match the loss dtype for decisions; use float64 only for drift summaries. + lp, olp, a = logp[m], old_lp[m], adv[m] p, mu, r = lp.exp(), olp.exp(), (lp - olp).exp() - dppo_masked = torch.where(a > 0, (p - mu) > delta, (p - mu) < -delta) + dppo_masked = ((a > 0) & ((p - mu) > delta)) | ((a < 0) & ((p - mu) < -delta)) grpo_clipped = ((a > 0) & (r > 1 + eps)) | ((a < 0) & (r < 1 - eps)) - absdp = (p - mu).abs() + drift = p.double() - mu.double() + absdp = drift.abs() + active = a != 0 return { "n_loss_tokens": float(n), + "n_nonzero_adv_tokens": int(active.sum()), + "dppo_removed_gradient_tokens": int(dppo_masked.sum()), + "dppo_masked_nonzero_adv_fraction": float(dppo_masked[active].double().mean()) if active.any() else None, + "policy_drift_abs_p_q99": float(torch.quantile(absdp, 0.99)), + "policy_drift_abs_p_max": float(absdp.max()), + "policy_drift_signed_p_min": float(drift.min()), + "policy_drift_signed_p_max": float(drift.max()), + "abs_drift_above_delta_tokens": int((absdp > delta).sum()), "dppo_masked_token_fraction": float(dppo_masked.double().mean()), "grpo_clip_fraction": float(grpo_clipped.double().mean()), "policy_drift_abs_p_q90": float(torch.quantile(absdp.float(), 0.9)),