Add model save/load and --resume for training continuation

- AsymmetricEncoder.save_checkpoint(): saves model_state + metadata
- AsymmetricEncoder.load_checkpoint(): rebuilds model with frozen backbones,
  then loads trainable weights from checkpoint
- --resume flag restores optimizer, loss (learnable tau), and scheduler state
- Training continues from the saved epoch

Usage:
  python -m src.training.train_gtauav --resume out/gtauav/with_text/ckpt_epoch004.pt

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pikaliov
2026-04-21 18:14:54 +03:00
parent 3014a5def8
commit 44bce3096c
2 changed files with 96 additions and 13 deletions

View File

@@ -527,6 +527,56 @@ class AsymmetricEncoder(nn.Module):
"""Return list of parameters that require grad."""
return [p for p in self.parameters() if p.requires_grad]
def save_checkpoint(self, path: str | Path, **extra) -> None:
"""Save model checkpoint with metadata."""
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
ckpt = {
"model_state": self.state_dict(),
"proj_dim": self.proj_dim,
"baseline_mode": self.baseline_mode,
**extra,
}
tmp = path.with_suffix(path.suffix + ".tmp")
torch.save(ckpt, tmp)
tmp.replace(path)
LOGGER.info("💾 Model saved to %s", path)
@classmethod
def load_checkpoint(
cls,
path: str | Path,
dino_web_path: str = "nn_models/DINO_WEB/dinov3-vitl16-pretrain-lvd1689m.pth",
dino_sat_path: str = "nn_models/DINO_SAT/model.safetensors",
lrsclip_path: str = "nn_models/LRSCLIP/DGTRS-CLIP-ViT-L-14.pt",
device: str = "cuda",
) -> tuple[AsymmetricEncoder, dict]:
"""Load model from checkpoint.
First builds the model (loading frozen backbones), then loads
the saved trainable weights on top.
Returns:
(model, checkpoint_dict) — model ready for eval/resume,
checkpoint_dict has optimizer_state, epoch, etc.
"""
path = Path(path)
LOGGER.info("📂 Loading checkpoint from %s", path)
ckpt = torch.load(str(path), map_location="cpu", weights_only=False)
model = cls(
dino_web_path=dino_web_path,
dino_sat_path=dino_sat_path,
lrsclip_path=lrsclip_path,
proj_dim=ckpt.get("proj_dim", 512),
baseline_mode=ckpt.get("baseline_mode", False),
device=device,
)
model.load_state_dict(ckpt["model_state"], strict=False)
model = model.to(device)
LOGGER.info("✅ Checkpoint loaded (epoch=%s)", ckpt.get("epoch", "?"))
return model, ckpt
def train(self, mode: bool = True) -> AsymmetricEncoder:
"""Override to keep frozen encoders in eval mode."""
super().train(mode)