Add CSV logging with pandas (train.csv, val.csv, per-epoch files)

Logs:
  {output_dir}/logs/train.csv — cumulative train metrics (all epochs)
  {output_dir}/logs/val.csv — cumulative val metrics (eval epochs)
  {output_dir}/logs/epoch_NNN_train.csv — per-epoch train
  {output_dir}/logs/epoch_NNN_val.csv — per-epoch val

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pikaliov
2026-04-21 19:46:07 +03:00
parent 2db3dff819
commit aee8212454

View File

@@ -16,6 +16,7 @@ from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
import coloredlogs import coloredlogs
import pandas as pd
import torch import torch
import torch.nn as nn import torch.nn as nn
from torch.amp import GradScaler, autocast from torch.amp import GradScaler, autocast
@@ -200,6 +201,36 @@ def _evaluate(
return metrics return metrics
class CSVLogger:
"""Log train/val metrics to CSV files using pandas.
Creates:
{output_dir}/logs/train.csv — all train epochs
{output_dir}/logs/val.csv — all val epochs
{output_dir}/logs/epoch_{N}.csv — per-epoch details
"""
def __init__(self, output_dir: Path) -> None:
self.log_dir = output_dir / "logs"
self.log_dir.mkdir(parents=True, exist_ok=True)
self.train_rows: list[dict] = []
self.val_rows: list[dict] = []
def log_train(self, epoch: int, metrics: dict, lr: float, elapsed: float) -> None:
row = {"epoch": epoch, "lr": lr, "elapsed_s": round(elapsed, 1), **metrics}
self.train_rows.append(row)
# Append to cumulative CSV.
pd.DataFrame(self.train_rows).to_csv(self.log_dir / "train.csv", index=False)
# Per-epoch CSV.
pd.DataFrame([row]).to_csv(self.log_dir / f"epoch_{epoch:03d}_train.csv", index=False)
def log_val(self, epoch: int, metrics: dict) -> None:
row = {"epoch": epoch, **metrics}
self.val_rows.append(row)
pd.DataFrame(self.val_rows).to_csv(self.log_dir / "val.csv", index=False)
pd.DataFrame([row]).to_csv(self.log_dir / f"epoch_{epoch:03d}_val.csv", index=False)
def _clear_vram() -> None: def _clear_vram() -> None:
"""Free VRAM from previous runs before starting.""" """Free VRAM from previous runs before starting."""
import gc import gc
@@ -354,6 +385,7 @@ def train(cfg: TrainConfigGTAUAV) -> None:
LOGGER.info("🔄 Resuming from epoch %d", start_epoch) LOGGER.info("🔄 Resuming from epoch %d", start_epoch)
history: list[dict] = [] history: list[dict] = []
csv_logger = CSVLogger(output_dir)
LOGGER.info("🚀 Starting training for %d epochs (from epoch %d)", cfg.epochs, start_epoch) LOGGER.info("🚀 Starting training for %d epochs (from epoch %d)", cfg.epochs, start_epoch)
@@ -442,10 +474,14 @@ def train(cfg: TrainConfigGTAUAV) -> None:
"train": means, "train": means,
} }
# Log train metrics to CSV.
csv_logger.log_train(epoch, means, optimizer.param_groups[0]["lr"], elapsed)
# Evaluation. # Evaluation.
if (epoch + 1) % cfg.eval_every == 0 or epoch == cfg.epochs - 1: if (epoch + 1) % cfg.eval_every == 0 or epoch == cfg.epochs - 1:
val_metrics = _evaluate(model, test_loader, cfg.device) val_metrics = _evaluate(model, test_loader, cfg.device)
epoch_record["val"] = val_metrics epoch_record["val"] = val_metrics
csv_logger.log_val(epoch, val_metrics)
LOGGER.info( LOGGER.info(
"🎯 val epoch=%d R@1=%.4f R@5=%.4f R@10=%.4f gate_q=%.4f gate_g=%.4f", "🎯 val epoch=%d R@1=%.4f R@5=%.4f R@10=%.4f gate_q=%.4f gate_g=%.4f",
epoch, epoch,