Add model save/load and --resume for training continuation
- AsymmetricEncoder.save_checkpoint(): saves model_state + metadata - AsymmetricEncoder.load_checkpoint(): rebuilds model with frozen backbones, then loads trainable weights from checkpoint - --resume flag restores optimizer, loss (learnable tau), and scheduler state - Training continues from the saved epoch Usage: python -m src.training.train_gtauav --resume out/gtauav/with_text/ckpt_epoch004.pt Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -527,6 +527,56 @@ class AsymmetricEncoder(nn.Module):
|
||||
"""Return list of parameters that require grad."""
|
||||
return [p for p in self.parameters() if p.requires_grad]
|
||||
|
||||
def save_checkpoint(self, path: str | Path, **extra) -> None:
|
||||
"""Save model checkpoint with metadata."""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
ckpt = {
|
||||
"model_state": self.state_dict(),
|
||||
"proj_dim": self.proj_dim,
|
||||
"baseline_mode": self.baseline_mode,
|
||||
**extra,
|
||||
}
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
torch.save(ckpt, tmp)
|
||||
tmp.replace(path)
|
||||
LOGGER.info("💾 Model saved to %s", path)
|
||||
|
||||
@classmethod
|
||||
def load_checkpoint(
|
||||
cls,
|
||||
path: str | Path,
|
||||
dino_web_path: str = "nn_models/DINO_WEB/dinov3-vitl16-pretrain-lvd1689m.pth",
|
||||
dino_sat_path: str = "nn_models/DINO_SAT/model.safetensors",
|
||||
lrsclip_path: str = "nn_models/LRSCLIP/DGTRS-CLIP-ViT-L-14.pt",
|
||||
device: str = "cuda",
|
||||
) -> tuple[AsymmetricEncoder, dict]:
|
||||
"""Load model from checkpoint.
|
||||
|
||||
First builds the model (loading frozen backbones), then loads
|
||||
the saved trainable weights on top.
|
||||
|
||||
Returns:
|
||||
(model, checkpoint_dict) — model ready for eval/resume,
|
||||
checkpoint_dict has optimizer_state, epoch, etc.
|
||||
"""
|
||||
path = Path(path)
|
||||
LOGGER.info("📂 Loading checkpoint from %s", path)
|
||||
ckpt = torch.load(str(path), map_location="cpu", weights_only=False)
|
||||
|
||||
model = cls(
|
||||
dino_web_path=dino_web_path,
|
||||
dino_sat_path=dino_sat_path,
|
||||
lrsclip_path=lrsclip_path,
|
||||
proj_dim=ckpt.get("proj_dim", 512),
|
||||
baseline_mode=ckpt.get("baseline_mode", False),
|
||||
device=device,
|
||||
)
|
||||
model.load_state_dict(ckpt["model_state"], strict=False)
|
||||
model = model.to(device)
|
||||
LOGGER.info("✅ Checkpoint loaded (epoch=%s)", ckpt.get("epoch", "?"))
|
||||
return model, ckpt
|
||||
|
||||
def train(self, mode: bool = True) -> AsymmetricEncoder:
|
||||
"""Override to keep frozen encoders in eval mode."""
|
||||
super().train(mode)
|
||||
|
||||
Reference in New Issue
Block a user