claude_refactor_v3: Add and passed test on splited gin-configs loading but without weigts

This commit is contained in:
pikaliov
2026-05-06 16:17:36 +03:00
parent 911d2ce4e6
commit 2362ce0adb
6 changed files with 525 additions and 1316 deletions

View File

@@ -230,7 +230,7 @@ class Trainer:
def train(self) -> None:
"""Full pipeline: setup → build → train → evaluate → cleanup."""
self._validate_backbone()
clear_vram()
#! clear_vram()
set_seed(self.pipeline_cfg.seed)
self._setup_output_dir()
self._setup_tracker()
@@ -256,6 +256,7 @@ class Trainer:
def _validate_backbone(self) -> None:
"""Reject unsupported backbones up front with a helpful message."""
LOGGER.info("⚙️ Validate backbone")
backbone = self.models_common_cfg.backbone
if backbone not in _SUPPORTED_BACKBONES:
raise NotImplementedError(
@@ -271,6 +272,7 @@ class Trainer:
def _setup_output_dir(self) -> None:
"""Create output_dir, save config.json, init csv_logger."""
LOGGER.info("⚙️ Setup out dir")
self.output_dir = Path(self.pipeline_cfg.output_dir)
self.output_dir.mkdir(parents=True, exist_ok=True)
@@ -290,6 +292,7 @@ class Trainer:
def _setup_tracker(self) -> None:
"""W&B + TensorBoard tracker."""
LOGGER.info("⚙️ Setup tracker...")
assert self.output_dir is not None and self.full_config is not None
self.tracker = ExperimentTracker(
output_dir=self.output_dir,
@@ -303,6 +306,7 @@ class Trainer:
def _build_model(self) -> None:
"""Build (or load) the encoder model based on the active backbone."""
LOGGER.info("⚙️ Build loss...")
backbone = self.models_common_cfg.backbone
if self.pipeline_cfg.resume_from is not None:
@@ -327,6 +331,7 @@ class Trainer:
def _build_model_from_resume(self, backbone: str) -> None:
"""Resume model from checkpoint. Sets self.model, self.resume_ckpt, self.start_epoch."""
LOGGER.info("⚙️ Build model from resume...")
LOGGER.info("Resuming from %s", self.pipeline_cfg.resume_from)
# Both DINOv3 and StripNet go through AsymmetricEncoder.load_checkpoint.
# Note: load_checkpoint doesn't support StripNet — known existing limitation.
@@ -348,6 +353,7 @@ class Trainer:
def _build_stripnet_model(self) -> nn.Module:
"""Construct AsymmetricEncoder configured for StripNet."""
LOGGER.info("⚙️ Build StripNet model...")
assert isinstance(self.models_cfg, StripNetModelsConfig)
m = self.models_cfg
# DINO paths passed but ignored at runtime when backbone='stripnet'.
@@ -370,6 +376,7 @@ class Trainer:
).to(self.hardware_cfg.device)
def _build_dinov3_model(self) -> nn.Module:
LOGGER.info("⚙️ Build DINOv3 model...")
"""Construct AsymmetricEncoder configured for DINOv3."""
assert isinstance(self.models_cfg, DINOv3ModelsConfig)
m = self.models_cfg
@@ -393,6 +400,7 @@ class Trainer:
def _configure_gradient_checkpointing(self) -> None:
"""Enable gradient checkpointing on encoders that support it."""
LOGGER.info("⚙️ Configure gradient checkpointing...")
assert self.model is not None
backbone = self.models_common_cfg.backbone
if not self.hardware_cfg.gradient_checkpointing:
@@ -406,11 +414,11 @@ class Trainer:
self.model.sat_encoder.set_gradient_checkpointing(True)
if self.model.text_encoder is not None:
self.model.text_encoder.transformer.gradient_checkpointing = True
LOGGER.info("Gradient checkpointing enabled (DINOv3 + DGTRS)")
LOGGER.info("Gradient checkpointing enabled (DINOv3 + DGTRS)")
elif backbone == "stripnet":
if self.model.text_encoder is not None:
self.model.text_encoder.transformer.gradient_checkpointing = True
LOGGER.info("Gradient checkpointing enabled (DGTRS only; StripNet doesn't support it)")
LOGGER.info("Gradient checkpointing enabled (DGTRS only; StripNet doesn't support it)")
def _log_model_summary(self) -> None:
"""Log trainable param count, save model_summary.txt, hook W&B."""
@@ -428,6 +436,7 @@ class Trainer:
def _build_loss(self) -> None:
"""Build InfoNCELoss or WeightedInfoNCELoss based on training_cfg.loss_type."""
LOGGER.info("⚙️ Build loss...")
t = self.training_cfg
if t.loss_type == "symmetric":
self.loss_fn = InfoNCELoss(
@@ -465,6 +474,7 @@ class Trainer:
def _build_neg_bank(self) -> None:
"""Optional NegativeMemoryBank for hard-negative mining."""
LOGGER.info("⚙️ Build negative bank...")
assert self.model is not None
if self.training_cfg.neg_bank_size > 0:
self.neg_bank = NegativeMemoryBank(
@@ -478,6 +488,7 @@ class Trainer:
def _build_data_loaders(self) -> None:
"""Build train/test/train_eval datasets, samplers, loaders."""
LOGGER.info("⚙️ Build dataloaders...")
drone_train_tf = get_drone_train_transform(image_size=256)
sat_train_tf = get_satellite_train_transform(image_size=256)
eval_tf = get_dino_transform(image_size=256)
@@ -594,6 +605,7 @@ class Trainer:
def _build_optimizer_and_scheduler(self) -> None:
"""Build AdamW with per-group LR + cosine-warmup scheduler + GradScaler."""
LOGGER.info("⚙️ Build optimizer & scheduler...")
assert self.model is not None and self.loss_fn is not None and self.train_loader is not None
t = self.training_cfg
@@ -637,6 +649,7 @@ class Trainer:
def _restore_from_resume(self) -> None:
"""Restore optimizer/scheduler/loss state on resume."""
LOGGER.info("⚙️ Restore from resume...")
if self.resume_ckpt is None:
return
assert self.optimizer is not None and self.loss_fn is not None and self.scheduler is not None
@@ -653,6 +666,7 @@ class Trainer:
def _setup_profiler(self) -> None:
"""Optional PyTorch profiler (only if start_epoch == 0)."""
LOGGER.info("⚙️ Setup profiler...")
if self.tracking_cfg.use_profiler and self.start_epoch == 0:
assert self.output_dir is not None
self.profiler = TrainingProfiler(