Fix NaN loss: revert MONA to fp32, fix loss logging
- MONA fp16 causes NaN (gamma=1e-6 underflows in fp16 min subnormal ~6e-8) - Revert MONA forward to fp32 with autocast(enabled=False), cast output back - Fix loss CSV: save raw_loss before backward() (tensor consumed after backward) - Verified: loss=3.78, no NaN, bs=48 peak=21.4 GB Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -540,6 +540,7 @@ def train(cfg: TrainConfigGTAUAV) -> None:
|
||||
)
|
||||
|
||||
# Scale loss by accumulation steps so gradients average correctly.
|
||||
raw_loss = float(loss_dict["total"].item()) # save before backward
|
||||
total_loss = loss_dict["total"] / accum
|
||||
scaler.scale(total_loss).backward()
|
||||
|
||||
@@ -568,7 +569,6 @@ def train(cfg: TrainConfigGTAUAV) -> None:
|
||||
global_step += 1
|
||||
|
||||
# --- Per-batch tracking (log unscaled loss) ---
|
||||
raw_loss = total_loss.item() * accum # undo /accum for logging
|
||||
step_metrics = {
|
||||
"loss": raw_loss,
|
||||
"temperature": float(loss_dict["temperature"].item()),
|
||||
@@ -585,9 +585,9 @@ def train(cfg: TrainConfigGTAUAV) -> None:
|
||||
|
||||
pbar.set_postfix(
|
||||
loss=f"{raw_loss:.3f}",
|
||||
tau=f"{loss_dict['temperature'].item():.4f}",
|
||||
gq=f"{loss_dict['gate_q'].item():.3f}",
|
||||
gg=f"{loss_dict['gate_g'].item():.3f}",
|
||||
tau=f"{step_metrics['temperature']:.4f}",
|
||||
gq=f"{step_metrics['gate_q']:.3f}",
|
||||
gg=f"{step_metrics['gate_g']:.3f}",
|
||||
)
|
||||
|
||||
# --- Profiler step ---
|
||||
|
||||
Reference in New Issue
Block a user