Remove projections (1024 native), add satellite text, dual GatedFusion

Architecture changes:
- Removed proj_drone/proj_sat (1024→512): retrieval space is now
  DINOv3 native 1024-dim, no information loss from projection
- TextFusionMLP: 2304→1024→1024 (was 2304→768→512), shared between branches
- Gallery branch now uses satellite captions (L1/L2/L3) via shared TextFusionMLP
- Two separate GatedFusion gates: α_q (query) and α_g (gallery)
- For sat images without captions (~57%): gate passes image features through

Dataset changes:
- GTAUAVDataset now loads satellite captions from caption index
- collate_gtauav_batch includes sat_caption_l1/l2/l3

Training loop:
- Passes satellite captions to model forward
- Logs both gate_q and gate_g values

11.1M trainable / 734M total (1.51%)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pikaliov
2026-04-21 19:01:30 +03:00
parent 219bb779eb
commit 0c41c1f017
6 changed files with 172 additions and 120 deletions

View File

@@ -61,7 +61,6 @@ class TrainConfigGTAUAV:
dino_web_path: str = _DINO_WEB
dino_sat_path: str = _DINO_SAT
lrsclip_path: str = _LRSCLIP
proj_dim: int = 512
init_gate: float = 0.7
baseline_mode: bool = False
@@ -169,6 +168,9 @@ def _evaluate(
caption_l1=batch["caption_l1"],
caption_l2=batch["caption_l2"],
caption_l3=batch["caption_l3"],
sat_caption_l1=batch["sat_caption_l1"],
sat_caption_l2=batch["sat_caption_l2"],
sat_caption_l3=batch["sat_caption_l3"],
)
all_query.append(embeddings["query"].cpu())
all_gallery.append(embeddings["gallery"].cpu())
@@ -193,7 +195,8 @@ def _evaluate(
hit = (top_k == targets.unsqueeze(1)).any(dim=1).float()
metrics[f"r@{k}_g2q"] = float(hit.mean().item())
metrics["gate"] = model.fusion.gate_value
metrics["gate_q"] = model.fusion_query.gate_value
metrics["gate_g"] = model.fusion_gallery.gate_value
return metrics
@@ -233,7 +236,6 @@ def train(cfg: TrainConfigGTAUAV) -> None:
dino_web_path=cfg.dino_web_path,
dino_sat_path=cfg.dino_sat_path,
lrsclip_path=cfg.lrsclip_path,
proj_dim=cfg.proj_dim,
init_gate=cfg.init_gate,
baseline_mode=cfg.baseline_mode,
device=cfg.device,
@@ -372,6 +374,9 @@ def train(cfg: TrainConfigGTAUAV) -> None:
caption_l1=batch["caption_l1"],
caption_l2=batch["caption_l2"],
caption_l3=batch["caption_l3"],
sat_caption_l1=batch["sat_caption_l1"],
sat_caption_l2=batch["sat_caption_l2"],
sat_caption_l3=batch["sat_caption_l3"],
)
# Loss in fp32 (learnable temperature gradient overflows in fp16).
loss_dict = loss_fn(
@@ -402,19 +407,21 @@ def train(cfg: TrainConfigGTAUAV) -> None:
pbar.set_postfix(
loss=f"{total_loss.item():.3f}",
tau=f"{loss_dict['temperature'].item():.4f}",
gate=f"{loss_dict['gate'].item():.3f}",
gq=f"{loss_dict['gate_q'].item():.3f}",
gg=f"{loss_dict['gate_g'].item():.3f}",
)
elapsed = time.time() - epoch_start
means = {k: v / max(n_batches, 1) for k, v in agg.items()}
LOGGER.info(
"📈 epoch=%d time=%.1fs lr=%.2e loss=%.4f tau=%.4f gate=%.4f",
"📈 epoch=%d time=%.1fs lr=%.2e loss=%.4f tau=%.4f gate_q=%.4f gate_g=%.4f",
epoch, elapsed,
optimizer.param_groups[0]["lr"],
means.get("total", 0.0),
means.get("temperature", 0.0),
means.get("gate", 1.0),
means.get("gate_q", 1.0),
means.get("gate_g", 1.0),
)
epoch_record: dict = {
@@ -428,12 +435,13 @@ def train(cfg: TrainConfigGTAUAV) -> None:
val_metrics = _evaluate(model, test_loader, cfg.device)
epoch_record["val"] = val_metrics
LOGGER.info(
"🎯 val epoch=%d R@1=%.4f R@5=%.4f R@10=%.4f gate=%.4f",
"🎯 val epoch=%d R@1=%.4f R@5=%.4f R@10=%.4f gate_q=%.4f gate_g=%.4f",
epoch,
val_metrics.get("r@1_q2g", 0.0),
val_metrics.get("r@5_q2g", 0.0),
val_metrics.get("r@10_q2g", 0.0),
val_metrics.get("gate", 1.0),
val_metrics.get("gate_q", 1.0),
val_metrics.get("gate_g", 1.0),
)
history.append(epoch_record)
@@ -469,11 +477,12 @@ def train(cfg: TrainConfigGTAUAV) -> None:
LOGGER.info("✅ Training complete. Report: %s", report_path)
LOGGER.info(
"📊 Final — R@1=%.4f R@5=%.4f R@10=%.4f gate=%.4f",
"📊 Final — R@1=%.4f R@5=%.4f R@10=%.4f gate_q=%.4f gate_g=%.4f",
final_metrics.get("r@1_q2g", 0.0),
final_metrics.get("r@5_q2g", 0.0),
final_metrics.get("r@10_q2g", 0.0),
final_metrics.get("gate", 1.0),
final_metrics.get("gate_q", 1.0),
final_metrics.get("gate_g", 1.0),
)