From f382325ac6956cc724b419bc2cc59f42e0832df6 Mon Sep 17 00:00:00 2001 From: pikaliov Date: Wed, 22 Apr 2026 07:41:32 +0300 Subject: [PATCH] CSVLogger: load previous epochs on resume for continuous plots On init, load existing train.csv/val.csv so that epoch-level metrics and plots include the full training history after checkpoint resume. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/training/train_gtauav.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/training/train_gtauav.py b/src/training/train_gtauav.py index da04071..9f8c8a1 100644 --- a/src/training/train_gtauav.py +++ b/src/training/train_gtauav.py @@ -242,13 +242,25 @@ class CSVLogger: def __init__(self, output_dir: Path) -> None: self.log_dir = output_dir / "logs" self.log_dir.mkdir(parents=True, exist_ok=True) - self.train_rows: list[dict] = [] - self.val_rows: list[dict] = [] self._current_epoch: int = -1 self._batch_columns: list[str] | None = None self._cumulative_batch_path = self.log_dir / "train_batches.csv" self._epoch_batch_path: Path | None = None + # Load existing CSV data on resume (so plots show full history). + train_csv = self.log_dir / "train.csv" + val_csv = self.log_dir / "val.csv" + if train_csv.exists(): + self.train_rows = pd.read_csv(train_csv).to_dict("records") + LOGGER.info("CSVLogger: loaded %d previous train epochs", len(self.train_rows)) + else: + self.train_rows = [] + if val_csv.exists(): + self.val_rows = pd.read_csv(val_csv).to_dict("records") + LOGGER.info("CSVLogger: loaded %d previous val epochs", len(self.val_rows)) + else: + self.val_rows = [] + def log_batch(self, epoch: int, batch_idx: int, global_step: int, metrics: dict) -> None: """Log metrics for a single training batch. Writes to disk immediately.""" row = {"epoch": epoch, "batch": batch_idx, "global_step": global_step, **metrics}