Remove projections (1024 native), add satellite text, dual GatedFusion
Architecture changes: - Removed proj_drone/proj_sat (1024→512): retrieval space is now DINOv3 native 1024-dim, no information loss from projection - TextFusionMLP: 2304→1024→1024 (was 2304→768→512), shared between branches - Gallery branch now uses satellite captions (L1/L2/L3) via shared TextFusionMLP - Two separate GatedFusion gates: α_q (query) and α_g (gallery) - For sat images without captions (~57%): gate passes image features through Dataset changes: - GTAUAVDataset now loads satellite captions from caption index - collate_gtauav_batch includes sat_caption_l1/l2/l3 Training loop: - Passes satellite captions to model forward - Logs both gate_q and gate_g values 11.1M trainable / 734M total (1.51%) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -61,7 +61,6 @@ class TrainConfigGTAUAV:
|
||||
dino_web_path: str = _DINO_WEB
|
||||
dino_sat_path: str = _DINO_SAT
|
||||
lrsclip_path: str = _LRSCLIP
|
||||
proj_dim: int = 512
|
||||
init_gate: float = 0.7
|
||||
baseline_mode: bool = False
|
||||
|
||||
@@ -169,6 +168,9 @@ def _evaluate(
|
||||
caption_l1=batch["caption_l1"],
|
||||
caption_l2=batch["caption_l2"],
|
||||
caption_l3=batch["caption_l3"],
|
||||
sat_caption_l1=batch["sat_caption_l1"],
|
||||
sat_caption_l2=batch["sat_caption_l2"],
|
||||
sat_caption_l3=batch["sat_caption_l3"],
|
||||
)
|
||||
all_query.append(embeddings["query"].cpu())
|
||||
all_gallery.append(embeddings["gallery"].cpu())
|
||||
@@ -193,7 +195,8 @@ def _evaluate(
|
||||
hit = (top_k == targets.unsqueeze(1)).any(dim=1).float()
|
||||
metrics[f"r@{k}_g2q"] = float(hit.mean().item())
|
||||
|
||||
metrics["gate"] = model.fusion.gate_value
|
||||
metrics["gate_q"] = model.fusion_query.gate_value
|
||||
metrics["gate_g"] = model.fusion_gallery.gate_value
|
||||
return metrics
|
||||
|
||||
|
||||
@@ -233,7 +236,6 @@ def train(cfg: TrainConfigGTAUAV) -> None:
|
||||
dino_web_path=cfg.dino_web_path,
|
||||
dino_sat_path=cfg.dino_sat_path,
|
||||
lrsclip_path=cfg.lrsclip_path,
|
||||
proj_dim=cfg.proj_dim,
|
||||
init_gate=cfg.init_gate,
|
||||
baseline_mode=cfg.baseline_mode,
|
||||
device=cfg.device,
|
||||
@@ -372,6 +374,9 @@ def train(cfg: TrainConfigGTAUAV) -> None:
|
||||
caption_l1=batch["caption_l1"],
|
||||
caption_l2=batch["caption_l2"],
|
||||
caption_l3=batch["caption_l3"],
|
||||
sat_caption_l1=batch["sat_caption_l1"],
|
||||
sat_caption_l2=batch["sat_caption_l2"],
|
||||
sat_caption_l3=batch["sat_caption_l3"],
|
||||
)
|
||||
# Loss in fp32 (learnable temperature gradient overflows in fp16).
|
||||
loss_dict = loss_fn(
|
||||
@@ -402,19 +407,21 @@ def train(cfg: TrainConfigGTAUAV) -> None:
|
||||
pbar.set_postfix(
|
||||
loss=f"{total_loss.item():.3f}",
|
||||
tau=f"{loss_dict['temperature'].item():.4f}",
|
||||
gate=f"{loss_dict['gate'].item():.3f}",
|
||||
gq=f"{loss_dict['gate_q'].item():.3f}",
|
||||
gg=f"{loss_dict['gate_g'].item():.3f}",
|
||||
)
|
||||
|
||||
elapsed = time.time() - epoch_start
|
||||
|
||||
means = {k: v / max(n_batches, 1) for k, v in agg.items()}
|
||||
LOGGER.info(
|
||||
"📈 epoch=%d time=%.1fs lr=%.2e loss=%.4f tau=%.4f gate=%.4f",
|
||||
"📈 epoch=%d time=%.1fs lr=%.2e loss=%.4f tau=%.4f gate_q=%.4f gate_g=%.4f",
|
||||
epoch, elapsed,
|
||||
optimizer.param_groups[0]["lr"],
|
||||
means.get("total", 0.0),
|
||||
means.get("temperature", 0.0),
|
||||
means.get("gate", 1.0),
|
||||
means.get("gate_q", 1.0),
|
||||
means.get("gate_g", 1.0),
|
||||
)
|
||||
|
||||
epoch_record: dict = {
|
||||
@@ -428,12 +435,13 @@ def train(cfg: TrainConfigGTAUAV) -> None:
|
||||
val_metrics = _evaluate(model, test_loader, cfg.device)
|
||||
epoch_record["val"] = val_metrics
|
||||
LOGGER.info(
|
||||
"🎯 val epoch=%d R@1=%.4f R@5=%.4f R@10=%.4f gate=%.4f",
|
||||
"🎯 val epoch=%d R@1=%.4f R@5=%.4f R@10=%.4f gate_q=%.4f gate_g=%.4f",
|
||||
epoch,
|
||||
val_metrics.get("r@1_q2g", 0.0),
|
||||
val_metrics.get("r@5_q2g", 0.0),
|
||||
val_metrics.get("r@10_q2g", 0.0),
|
||||
val_metrics.get("gate", 1.0),
|
||||
val_metrics.get("gate_q", 1.0),
|
||||
val_metrics.get("gate_g", 1.0),
|
||||
)
|
||||
|
||||
history.append(epoch_record)
|
||||
@@ -469,11 +477,12 @@ def train(cfg: TrainConfigGTAUAV) -> None:
|
||||
|
||||
LOGGER.info("✅ Training complete. Report: %s", report_path)
|
||||
LOGGER.info(
|
||||
"📊 Final — R@1=%.4f R@5=%.4f R@10=%.4f gate=%.4f",
|
||||
"📊 Final — R@1=%.4f R@5=%.4f R@10=%.4f gate_q=%.4f gate_g=%.4f",
|
||||
final_metrics.get("r@1_q2g", 0.0),
|
||||
final_metrics.get("r@5_q2g", 0.0),
|
||||
final_metrics.get("r@10_q2g", 0.0),
|
||||
final_metrics.get("gate", 1.0),
|
||||
final_metrics.get("gate_q", 1.0),
|
||||
final_metrics.get("gate_g", 1.0),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user