CSVLogger: load previous epochs on resume for continuous plots
On init, load existing train.csv/val.csv so that epoch-level metrics and plots include the full training history after checkpoint resume. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -242,13 +242,25 @@ class CSVLogger:
|
||||
def __init__(self, output_dir: Path) -> None:
|
||||
self.log_dir = output_dir / "logs"
|
||||
self.log_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.train_rows: list[dict] = []
|
||||
self.val_rows: list[dict] = []
|
||||
self._current_epoch: int = -1
|
||||
self._batch_columns: list[str] | None = None
|
||||
self._cumulative_batch_path = self.log_dir / "train_batches.csv"
|
||||
self._epoch_batch_path: Path | None = None
|
||||
|
||||
# Load existing CSV data on resume (so plots show full history).
|
||||
train_csv = self.log_dir / "train.csv"
|
||||
val_csv = self.log_dir / "val.csv"
|
||||
if train_csv.exists():
|
||||
self.train_rows = pd.read_csv(train_csv).to_dict("records")
|
||||
LOGGER.info("CSVLogger: loaded %d previous train epochs", len(self.train_rows))
|
||||
else:
|
||||
self.train_rows = []
|
||||
if val_csv.exists():
|
||||
self.val_rows = pd.read_csv(val_csv).to_dict("records")
|
||||
LOGGER.info("CSVLogger: loaded %d previous val epochs", len(self.val_rows))
|
||||
else:
|
||||
self.val_rows = []
|
||||
|
||||
def log_batch(self, epoch: int, batch_idx: int, global_step: int, metrics: dict) -> None:
|
||||
"""Log metrics for a single training batch. Writes to disk immediately."""
|
||||
row = {"epoch": epoch, "batch": batch_idx, "global_step": global_step, **metrics}
|
||||
|
||||
Reference in New Issue
Block a user