Pikaliov/caption-test
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CSVLogger: load previous epochs on resume for continuous plots

Browse Source 
On init, load existing train.csv/val.csv so that epoch-level metrics
and plots include the full training history after checkpoint resume.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pikaliov
2026-04-22 07:41:32 +03:00
parent a32cb8ac6c
commit f382325ac6
1 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/training/train_gtauav.py
View File

@@ -242,13 +242,25 @@ class CSVLogger:
def __init__(self, output_dir: Path) -> None: def __init__(self, output_dir: Path) -> None:
self.log_dir = output_dir / "logs" self.log_dir = output_dir / "logs"
self.log_dir.mkdir(parents=True, exist_ok=True) self.log_dir.mkdir(parents=True, exist_ok=True)
self.train_rows: list[dict] = []
self.val_rows: list[dict] = []
self._current_epoch: int = -1 self._current_epoch: int = -1
self._batch_columns: list[str] | None = None self._batch_columns: list[str] | None = None
self._cumulative_batch_path = self.log_dir / "train_batches.csv" self._cumulative_batch_path = self.log_dir / "train_batches.csv"
self._epoch_batch_path: Path | None = None self._epoch_batch_path: Path | None = None
# Load existing CSV data on resume (so plots show full history).
train_csv = self.log_dir / "train.csv"
val_csv = self.log_dir / "val.csv"
if train_csv.exists():
self.train_rows = pd.read_csv(train_csv).to_dict("records")
LOGGER.info("CSVLogger: loaded %d previous train epochs", len(self.train_rows))
else:
self.train_rows = []
if val_csv.exists():
self.val_rows = pd.read_csv(val_csv).to_dict("records")
LOGGER.info("CSVLogger: loaded %d previous val epochs", len(self.val_rows))
else:
self.val_rows = []
def log_batch(self, epoch: int, batch_idx: int, global_step: int, metrics: dict) -> None: def log_batch(self, epoch: int, batch_idx: int, global_step: int, metrics: dict) -> None:
"""Log metrics for a single training batch. Writes to disk immediately.""" """Log metrics for a single training batch. Writes to disk immediately."""
row = {"epoch": epoch, "batch": batch_idx, "global_step": global_step, **metrics} row = {"epoch": epoch, "batch": batch_idx, "global_step": global_step, **metrics}