claude_refactor_v3: Add and passed test on splited gin-configs loading but without weigts
This commit is contained in:
@@ -230,7 +230,7 @@ class Trainer:
|
||||
def train(self) -> None:
|
||||
"""Full pipeline: setup → build → train → evaluate → cleanup."""
|
||||
self._validate_backbone()
|
||||
clear_vram()
|
||||
#! clear_vram()
|
||||
set_seed(self.pipeline_cfg.seed)
|
||||
self._setup_output_dir()
|
||||
self._setup_tracker()
|
||||
@@ -256,6 +256,7 @@ class Trainer:
|
||||
|
||||
def _validate_backbone(self) -> None:
|
||||
"""Reject unsupported backbones up front with a helpful message."""
|
||||
LOGGER.info("⚙️ Validate backbone")
|
||||
backbone = self.models_common_cfg.backbone
|
||||
if backbone not in _SUPPORTED_BACKBONES:
|
||||
raise NotImplementedError(
|
||||
@@ -271,6 +272,7 @@ class Trainer:
|
||||
|
||||
def _setup_output_dir(self) -> None:
|
||||
"""Create output_dir, save config.json, init csv_logger."""
|
||||
LOGGER.info("⚙️ Setup out dir")
|
||||
self.output_dir = Path(self.pipeline_cfg.output_dir)
|
||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -290,6 +292,7 @@ class Trainer:
|
||||
|
||||
def _setup_tracker(self) -> None:
|
||||
"""W&B + TensorBoard tracker."""
|
||||
LOGGER.info("⚙️ Setup tracker...")
|
||||
assert self.output_dir is not None and self.full_config is not None
|
||||
self.tracker = ExperimentTracker(
|
||||
output_dir=self.output_dir,
|
||||
@@ -303,6 +306,7 @@ class Trainer:
|
||||
|
||||
def _build_model(self) -> None:
|
||||
"""Build (or load) the encoder model based on the active backbone."""
|
||||
LOGGER.info("⚙️ Build loss...")
|
||||
backbone = self.models_common_cfg.backbone
|
||||
|
||||
if self.pipeline_cfg.resume_from is not None:
|
||||
@@ -327,6 +331,7 @@ class Trainer:
|
||||
|
||||
def _build_model_from_resume(self, backbone: str) -> None:
|
||||
"""Resume model from checkpoint. Sets self.model, self.resume_ckpt, self.start_epoch."""
|
||||
LOGGER.info("⚙️ Build model from resume...")
|
||||
LOGGER.info("Resuming from %s", self.pipeline_cfg.resume_from)
|
||||
# Both DINOv3 and StripNet go through AsymmetricEncoder.load_checkpoint.
|
||||
# Note: load_checkpoint doesn't support StripNet — known existing limitation.
|
||||
@@ -348,6 +353,7 @@ class Trainer:
|
||||
|
||||
def _build_stripnet_model(self) -> nn.Module:
|
||||
"""Construct AsymmetricEncoder configured for StripNet."""
|
||||
LOGGER.info("⚙️ Build StripNet model...")
|
||||
assert isinstance(self.models_cfg, StripNetModelsConfig)
|
||||
m = self.models_cfg
|
||||
# DINO paths passed but ignored at runtime when backbone='stripnet'.
|
||||
@@ -370,6 +376,7 @@ class Trainer:
|
||||
).to(self.hardware_cfg.device)
|
||||
|
||||
def _build_dinov3_model(self) -> nn.Module:
|
||||
LOGGER.info("⚙️ Build DINOv3 model...")
|
||||
"""Construct AsymmetricEncoder configured for DINOv3."""
|
||||
assert isinstance(self.models_cfg, DINOv3ModelsConfig)
|
||||
m = self.models_cfg
|
||||
@@ -393,6 +400,7 @@ class Trainer:
|
||||
|
||||
def _configure_gradient_checkpointing(self) -> None:
|
||||
"""Enable gradient checkpointing on encoders that support it."""
|
||||
LOGGER.info("⚙️ Configure gradient checkpointing...")
|
||||
assert self.model is not None
|
||||
backbone = self.models_common_cfg.backbone
|
||||
if not self.hardware_cfg.gradient_checkpointing:
|
||||
@@ -406,11 +414,11 @@ class Trainer:
|
||||
self.model.sat_encoder.set_gradient_checkpointing(True)
|
||||
if self.model.text_encoder is not None:
|
||||
self.model.text_encoder.transformer.gradient_checkpointing = True
|
||||
LOGGER.info("Gradient checkpointing enabled (DINOv3 + DGTRS)")
|
||||
LOGGER.info("✅ Gradient checkpointing enabled (DINOv3 + DGTRS)")
|
||||
elif backbone == "stripnet":
|
||||
if self.model.text_encoder is not None:
|
||||
self.model.text_encoder.transformer.gradient_checkpointing = True
|
||||
LOGGER.info("Gradient checkpointing enabled (DGTRS only; StripNet doesn't support it)")
|
||||
LOGGER.info("✅ Gradient checkpointing enabled (DGTRS only; StripNet doesn't support it)")
|
||||
|
||||
def _log_model_summary(self) -> None:
|
||||
"""Log trainable param count, save model_summary.txt, hook W&B."""
|
||||
@@ -428,6 +436,7 @@ class Trainer:
|
||||
|
||||
def _build_loss(self) -> None:
|
||||
"""Build InfoNCELoss or WeightedInfoNCELoss based on training_cfg.loss_type."""
|
||||
LOGGER.info("⚙️ Build loss...")
|
||||
t = self.training_cfg
|
||||
if t.loss_type == "symmetric":
|
||||
self.loss_fn = InfoNCELoss(
|
||||
@@ -465,6 +474,7 @@ class Trainer:
|
||||
|
||||
def _build_neg_bank(self) -> None:
|
||||
"""Optional NegativeMemoryBank for hard-negative mining."""
|
||||
LOGGER.info("⚙️ Build negative bank...")
|
||||
assert self.model is not None
|
||||
if self.training_cfg.neg_bank_size > 0:
|
||||
self.neg_bank = NegativeMemoryBank(
|
||||
@@ -478,6 +488,7 @@ class Trainer:
|
||||
|
||||
def _build_data_loaders(self) -> None:
|
||||
"""Build train/test/train_eval datasets, samplers, loaders."""
|
||||
LOGGER.info("⚙️ Build dataloaders...")
|
||||
drone_train_tf = get_drone_train_transform(image_size=256)
|
||||
sat_train_tf = get_satellite_train_transform(image_size=256)
|
||||
eval_tf = get_dino_transform(image_size=256)
|
||||
@@ -594,6 +605,7 @@ class Trainer:
|
||||
|
||||
def _build_optimizer_and_scheduler(self) -> None:
|
||||
"""Build AdamW with per-group LR + cosine-warmup scheduler + GradScaler."""
|
||||
LOGGER.info("⚙️ Build optimizer & scheduler...")
|
||||
assert self.model is not None and self.loss_fn is not None and self.train_loader is not None
|
||||
t = self.training_cfg
|
||||
|
||||
@@ -637,6 +649,7 @@ class Trainer:
|
||||
|
||||
def _restore_from_resume(self) -> None:
|
||||
"""Restore optimizer/scheduler/loss state on resume."""
|
||||
LOGGER.info("⚙️ Restore from resume...")
|
||||
if self.resume_ckpt is None:
|
||||
return
|
||||
assert self.optimizer is not None and self.loss_fn is not None and self.scheduler is not None
|
||||
@@ -653,6 +666,7 @@ class Trainer:
|
||||
|
||||
def _setup_profiler(self) -> None:
|
||||
"""Optional PyTorch profiler (only if start_epoch == 0)."""
|
||||
LOGGER.info("⚙️ Setup profiler...")
|
||||
if self.tracking_cfg.use_profiler and self.start_epoch == 0:
|
||||
assert self.output_dir is not None
|
||||
self.profiler = TrainingProfiler(
|
||||
|
||||
Reference in New Issue
Block a user