Fix GTA-UAV eval + training pipeline: full gallery, mutex sampler, per-sample mask
Six critical fixes to the caption-test training/eval stack:
1. **IndentationError blocker** (train_gtauav.py:765-766)
Unparseable file — train-recall LOGGER.info block was orphaned outside
its `if eval_every` guard. Wrapped in `if train_recall:` so val eval
and Grad-CAM only run on eval epochs.
2. **Full satellite gallery in `_evaluate`**
Old code assembled gallery from DataLoader batches (one random sat per
drone), producing an incomplete gallery of size ≈ N_query instead of
N_unique_sat. Metrics were inflated because retrieval was against a
subset that always contained the target.
New `GTAUAVSatGallery` / `GTAUAVDroneQuery` iterate all unique tiles
and queries independently; full-gallery multi-match R@K + MRR.
3. **Per-sample caption mask** (`AsymmetricEncoder._fuse_with_mask`)
Mixed batches (some samples have captions, some don't) previously
encoded empty strings through DGTRS and mixed the noise output into
every sample via scalar gate. New `encode_query`/`encode_gallery` use
`torch.where` to fall back to pure image features for empty-caption
samples. Training `forward()` routes through the same helper so
training and eval share code.
4. **Symmetric InfoNCE as primary loss** (multi_infonce.InfoNCELoss)
Switched gin default from `WeightedInfoNCELoss` (adaptive label
smoothing — not the Game4Loc soft-IoU target it claimed) to the
existing symmetric InfoNCE with q2g=0.6/g2q=0.4 weighting. Loss type
now selectable via `cfg.loss_type ∈ {"symmetric", "weighted"}`.
5. **MutuallyExclusiveSampler** (new file)
BatchSampler that greedily packs drones whose `sat_candidates` sets
are pairwise disjoint within a batch. Eliminates false negatives from
the semi-positive graph without needing soft-label losses.
At bs=8 keeps 100% of 24,891 train entries; at bs=64 keeps 92.6%.
`set_epoch()` for reproducibility + different batches per epoch.
6. **Temperature clamp [0.01, 0.1]** (both loss modules)
Old tau_max=0.5 allowed the logit distribution to collapse into a
near-uniform softmax. Tightened to the CLIP-standard range.
Also:
- Added `scripts/smoke_eval.py` / `scripts/smoke_train.py` for fast
regression checks (eval in ~2 min, 2 train steps in ~1 min on RTX 4090).
- CLAUDE.md updated to reflect the new pipeline.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -371,7 +371,8 @@ class AsymmetricEncoder(nn.Module):
|
||||
|
||||
Returns None if all captions are empty (no text available).
|
||||
For mixed batches (some have captions, some don't), encodes all
|
||||
and lets GatedFusion handle per-sample gating.
|
||||
texts (empty strings tokenize to pad+EOS — their outputs must be
|
||||
masked downstream, see `_fuse_with_mask`).
|
||||
"""
|
||||
# Check if any caption is non-empty.
|
||||
if all(t == "" for t in l1_texts):
|
||||
@@ -388,6 +389,74 @@ class AsymmetricEncoder(nn.Module):
|
||||
tokens = tokenize_dgtrs(list(texts)).to(self.device)
|
||||
return self.text_encoder(tokens)
|
||||
|
||||
def _fuse_with_mask(
|
||||
self,
|
||||
img_feat: torch.Tensor,
|
||||
l1_texts: list[str] | None,
|
||||
l2_texts: list[str] | None,
|
||||
l3_texts: list[str] | None,
|
||||
fusion: GatedFusion,
|
||||
) -> torch.Tensor:
|
||||
"""Fuse image features with optional text, respecting per-sample presence.
|
||||
|
||||
For samples where caption is an empty string, output falls back to
|
||||
pure image features (avoiding noise contamination from empty-string
|
||||
text embeddings). For samples with captions, applies the standard
|
||||
gated fusion `σ(α)·img + (1-σ(α))·text`.
|
||||
|
||||
Returns L2-normalized [B, D] embedding.
|
||||
"""
|
||||
if (
|
||||
self.baseline_mode
|
||||
or l1_texts is None
|
||||
or l2_texts is None
|
||||
or l3_texts is None
|
||||
):
|
||||
return F.normalize(fusion(img_feat, None), dim=-1)
|
||||
|
||||
has_text = torch.tensor(
|
||||
[t != "" for t in l1_texts], dtype=torch.bool, device=img_feat.device,
|
||||
)
|
||||
if not has_text.any():
|
||||
return F.normalize(fusion(img_feat, None), dim=-1)
|
||||
|
||||
z_text = self.encode_text_levels(l1_texts, l2_texts, l3_texts)
|
||||
if z_text is None:
|
||||
return F.normalize(fusion(img_feat, None), dim=-1)
|
||||
|
||||
# Per-sample fusion: text-present samples use full gated fusion,
|
||||
# empty-caption samples pass through pure image features.
|
||||
gate = torch.sigmoid(fusion.alpha)
|
||||
fused_with_text = gate * img_feat + (1.0 - gate) * z_text
|
||||
out = torch.where(has_text.unsqueeze(-1), fused_with_text, img_feat)
|
||||
return F.normalize(out, dim=-1)
|
||||
|
||||
def encode_query(
|
||||
self,
|
||||
drone_img: torch.Tensor,
|
||||
caption_l1: list[str] | None = None,
|
||||
caption_l2: list[str] | None = None,
|
||||
caption_l3: list[str] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Encode drone → normalized query embedding with per-sample text mask."""
|
||||
drone_feat = self.encode_drone(drone_img)
|
||||
return self._fuse_with_mask(
|
||||
drone_feat, caption_l1, caption_l2, caption_l3, self.fusion_query,
|
||||
)
|
||||
|
||||
def encode_gallery(
|
||||
self,
|
||||
sat_img: torch.Tensor,
|
||||
sat_caption_l1: list[str] | None = None,
|
||||
sat_caption_l2: list[str] | None = None,
|
||||
sat_caption_l3: list[str] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Encode satellite → normalized gallery embedding with per-sample text mask."""
|
||||
sat_feat = self.encode_satellite(sat_img)
|
||||
return self._fuse_with_mask(
|
||||
sat_feat, sat_caption_l1, sat_caption_l2, sat_caption_l3, self.fusion_gallery,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
drone_img: torch.Tensor,
|
||||
@@ -401,6 +470,10 @@ class AsymmetricEncoder(nn.Module):
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Forward pass.
|
||||
|
||||
Both branches use per-sample caption masking: samples with an empty
|
||||
caption string fall back to pure image features instead of being
|
||||
fused with noise from empty-string text embeddings.
|
||||
|
||||
Args:
|
||||
drone_img: Drone images [B, 3, 256, 256].
|
||||
sat_img: Satellite images [B, 3, 256, 256].
|
||||
@@ -411,28 +484,8 @@ class AsymmetricEncoder(nn.Module):
|
||||
Dict with 'query' [B, embed_dim], 'gallery' [B, embed_dim],
|
||||
'gate_q', 'gate_g'.
|
||||
"""
|
||||
# Image features (frozen DINOv3).
|
||||
drone_feat = self.encode_drone(drone_img)
|
||||
sat_feat = self.encode_satellite(sat_img)
|
||||
|
||||
# Query branch: drone + drone text.
|
||||
drone_text = None
|
||||
if (caption_l1 is not None and caption_l2 is not None
|
||||
and caption_l3 is not None and not self.baseline_mode):
|
||||
drone_text = self.encode_text_levels(caption_l1, caption_l2, caption_l3)
|
||||
|
||||
query = self.fusion_query(drone_feat, drone_text)
|
||||
query = F.normalize(query, dim=-1)
|
||||
|
||||
# Gallery branch: satellite + satellite text.
|
||||
sat_text = None
|
||||
if (sat_caption_l1 is not None and sat_caption_l2 is not None
|
||||
and sat_caption_l3 is not None and not self.baseline_mode):
|
||||
sat_text = self.encode_text_levels(sat_caption_l1, sat_caption_l2, sat_caption_l3)
|
||||
|
||||
gallery = self.fusion_gallery(sat_feat, sat_text)
|
||||
gallery = F.normalize(gallery, dim=-1)
|
||||
|
||||
query = self.encode_query(drone_img, caption_l1, caption_l2, caption_l3)
|
||||
gallery = self.encode_gallery(sat_img, sat_caption_l1, sat_caption_l2, sat_caption_l3)
|
||||
return {
|
||||
"query": query,
|
||||
"gallery": gallery,
|
||||
|
||||
Reference in New Issue
Block a user