From 814586ce3be7310425f5e200ce67c5f2a46150e1 Mon Sep 17 00:00:00 2001 From: pikaliov Date: Sat, 25 Apr 2026 13:47:02 +0300 Subject: [PATCH] Add g2q eval metrics and asymmetric MONA-24 configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _evaluate: compute R@K + AP for both directions (q2g and g2q) via inverted ground truth; g2q denominator counts only sat-tiles with at least one positive drone in the (sub)sampled query set. Surfaces in train.csv, val.csv, train_recall.csv, W&B summary, and final log. - conf/gtauav_balanced_asym.gin: asymmetric WEB+SAT encoders, MONA in all 24 ViT blocks (~17.6M trainable / ~733M total). - conf/gtauav_baseline_asym.gin: same architecture, baseline_mode=True for Δ R@1 against balanced_asym. - CLAUDE.md / README.md: document new configs, clarify that g2q is now computed (was claimed but missing). Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 14 +++++--- README.md | 29 +++++++++++++-- conf/gtauav_balanced_asym.gin | 16 +++++++++ conf/gtauav_baseline_asym.gin | 9 +++++ src/training/train_gtauav.py | 67 ++++++++++++++++++++++++++++++++--- 5 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 conf/gtauav_balanced_asym.gin create mode 100644 conf/gtauav_baseline_asym.gin diff --git a/CLAUDE.md b/CLAUDE.md index 36d64bd..3bc2d57 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,8 +118,10 @@ Eval: Resize(256) + CenterCrop(256) + ImageNet normalization. | `src/training/gradcam.py` | Grad-CAM visualization для DINOv3 encoders | | `src/training/profiling.py` | PyTorch Profiler wrapper + torchinfo model summary | | `src/training/plot_metrics.py` | Seaborn/matplotlib plots (каждую эпоху) | -| `conf/gtauav_balanced.gin` | With text, gate=0.7, 10 epochs | -| `conf/gtauav_baseline.gin` | No text, gate=1.0 | +| `conf/gtauav_balanced.gin` | Shared encoder, MONA 12/24, with text, gate=0.7, 10 epochs | +| `conf/gtauav_baseline.gin` | Shared encoder, MONA 12/24, no text, gate=1.0 | +| `conf/gtauav_balanced_asym.gin` | Asymmetric (WEB+SAT), MONA 24/24, with text — overrides gtauav_balanced.gin | +| `conf/gtauav_baseline_asym.gin` | Asymmetric (WEB+SAT), MONA 24/24, no text | | `conf/gtauav_text_heavy.gin` | Text-heavy, gate=0.3 | | `conf/gtauav_image_heavy.gin` | Image-heavy, gate=0.9 | | `scripts/make_split.py` | 80/20 random split из всех пар | @@ -228,8 +230,10 @@ Meta-файл `meta/seg_filter.json`: исключение изображени ### V3 (GTA-UAV, gin) | Конфиг | Gate init | Описание | |--------|-----------|----------| -| `conf/gtauav_balanced.gin` | 0.7 (30% text) | **Primary test** | -| `conf/gtauav_baseline.gin` | 1.0 (no text) | Reference baseline | +| `conf/gtauav_balanced.gin` | 0.7 (30% text) | **Primary test** — shared DINOv3 WEB, MONA 12/24 | +| `conf/gtauav_baseline.gin` | 1.0 (no text) | Reference baseline (shared, MONA 12/24) | +| `conf/gtauav_balanced_asym.gin` | 0.7 (30% text) | Asymmetric (WEB+SAT), MONA 24/24 — full-capacity variant | +| `conf/gtauav_baseline_asym.gin` | 1.0 (no text) | Asymmetric baseline for Δ R@1 vs balanced_asym | | `conf/gtauav_text_heavy.gin` | 0.3 (70% text) | Stress test | | `conf/gtauav_image_heavy.gin` | 0.9 (10% text) | Image-dominant | @@ -295,7 +299,7 @@ python -m scripts.compare_runs \ | 0 to +1% | WEAK — redesign caption pipeline | | < 0 | HARMFUL — critical bug | -**Eval metrics:** R@1, R@5, R@10 для drone->satellite и satellite->drone +**Eval metrics:** R@1, R@5, R@10 + AP (MRR) для обоих направлений: q2g (drone→satellite) и g2q (satellite→drone). g2q считается через инвертированный GT (для каждого sat-tile собираются drone-индексы из `valid_idx_per_query`); знаменатель — sat-tiles, у которых есть хотя бы один positive drone в (под)выборке. **Splits (GTA-UAV):** cross-area (primary, harder) и same-area (sanity check) **Logged per epoch:** loss, temperature (tau), gate value (sigma(alpha)), lr diff --git a/README.md b/README.md index 9a6b286..9d6f73b 100644 --- a/README.md +++ b/README.md @@ -355,8 +355,22 @@ Gradient checkpointing: DINOv3 (24 blocks) + DGTRS-CLIP (12 blocks) | logit_scale | 1 | 1 | learnable temperature | | **Total (shared)** | **432M** | **5.6M (1.30%)** | retrieval dim = 512 | -> **Asymmetric mode** (`--shared-encoder false`): uses separate DINOv3 WEB (drone) + DINOv3 SAT +> **Asymmetric mode** (`shared_encoder=False`): separate DINOv3 WEB (drone) + DINOv3 SAT > (satellite) encoders with independent MONA adapters. Requires ~4-5 GB more VRAM. +> Use `conf/gtauav_balanced_asym.gin` / `conf/gtauav_baseline_asym.gin` — these set +> `shared_encoder=False` and `mona_last_n_blocks=24` for the full-capacity setup. + +### Eval directions + +`_evaluate` computes R@1/5/10 and AP (MRR) for both retrieval directions: + +| Key | Direction | Notes | +|-----|-----------|-------| +| `r@K_q2g`, `ap_q2g` | drone → satellite (query → gallery) | Denominator: all queries (q2g convention) | +| `r@K_g2q`, `ap_g2q` | satellite → drone (gallery → query) | Denominator: only sat-tiles with ≥1 positive drone in subsample | + +g2q is computed via inverted ground truth: for each sat-tile, collect the drone indices +that list it as a valid candidate. `n_scored_g2q` is reported in metrics for transparency. ## Experiments @@ -385,8 +399,10 @@ Gallery: sat_img -> GeoRSCLIP -> gallery ``` caption-test/ ├── conf/ # Gin configs -│ ├── gtauav_balanced.gin # GTA-UAV with text (10 epochs, v3) -│ ├── gtauav_baseline.gin # GTA-UAV baseline, no text (v3) +│ ├── gtauav_balanced.gin # GTA-UAV with text, shared encoder, MONA 12/24 (v3) +│ ├── gtauav_baseline.gin # GTA-UAV baseline, shared, MONA 12/24, no text (v3) +│ ├── gtauav_balanced_asym.gin # GTA-UAV with text, asymmetric WEB+SAT, MONA 24/24 +│ ├── gtauav_baseline_asym.gin # GTA-UAV baseline, asymmetric, MONA 24/24, no text │ ├── gtauav_text_heavy.gin # GTA-UAV text-heavy gate=0.3 (v3) │ ├── gtauav_image_heavy.gin # GTA-UAV image-heavy gate=0.9 (v3) │ ├── balanced.gin # UAV-GeoLoc with text (v2) @@ -485,6 +501,13 @@ python -m src.training.train_gtauav --config conf/gtauav_text_heavy.gin \ # Image-heavy (gate=0.9, 10% text weight) python -m src.training.train_gtauav --config conf/gtauav_image_heavy.gin \ --filter-meta meta/seg_filter.json --batch-size 48 --epochs 30 + +# Asymmetric variants: separate WEB (drone) + SAT (satellite) encoders, MONA in all 24 blocks +# Higher capacity (~733M total / ~17.6M trainable), larger VRAM footprint. +python -m src.training.train_gtauav --config conf/gtauav_balanced_asym.gin \ + --filter-meta meta/seg_filter.json +python -m src.training.train_gtauav --config conf/gtauav_baseline_asym.gin \ + --filter-meta meta/seg_filter.json ``` ### 3. Train without gin (CLI-only) diff --git a/conf/gtauav_balanced_asym.gin b/conf/gtauav_balanced_asym.gin new file mode 100644 index 0000000..752bccd --- /dev/null +++ b/conf/gtauav_balanced_asym.gin @@ -0,0 +1,16 @@ +# GTA-UAV Balanced (asymmetric, full MONA): WEB drone encoder + SAT satellite encoder. +# MONA injected into all 24 ViT blocks of each encoder. +# Same loss/sampling/optimizer as gtauav_balanced.gin; differs only in model arch. +# +# Trainable: ~17.6M (MONA 2× × 24 blocks + LoRA + TextFusionMLP + gates + tau) +# Total params: ~733M (2× DINOv3-L + DGTRS-CLIP) +# VRAM target (RTX 4090, 24 GB): ~16-20 GB at batch=8 with gradient checkpointing. + +include 'conf/gtauav_balanced.gin' + +# ---- Model overrides: asymmetric + full MONA ---- +TrainConfigGTAUAV.shared_encoder = False # WEB for drone, SAT for satellite +TrainConfigGTAUAV.mona_last_n_blocks = 24 # MONA in all 24 ViT blocks (was 12) + +# ---- Output ---- +TrainConfigGTAUAV.output_dir = "out/gtauav/balanced_asym" diff --git a/conf/gtauav_baseline_asym.gin b/conf/gtauav_baseline_asym.gin new file mode 100644 index 0000000..c5ddd36 --- /dev/null +++ b/conf/gtauav_baseline_asym.gin @@ -0,0 +1,9 @@ +# GTA-UAV Baseline (asymmetric, full MONA): no text fusion (gate forced to 1.0). +# WEB drone encoder + SAT satellite encoder, MONA in all 24 ViT blocks. +# Reference R@1 for delta computation against gtauav_balanced_asym.gin. + +include 'conf/gtauav_balanced_asym.gin' + +TrainConfigGTAUAV.baseline_mode = True +TrainConfigGTAUAV.output_dir = "out/gtauav/baseline_asym" +TrainConfigGTAUAV.use_gradcam = False diff --git a/src/training/train_gtauav.py b/src/training/train_gtauav.py index 7127022..6590f68 100644 --- a/src/training/train_gtauav.py +++ b/src/training/train_gtauav.py @@ -382,8 +382,42 @@ def _evaluate( mrr_sum += 1.0 / (rank + 1) break metrics["ap_q2g"] = mrr_sum / max(n_scored, 1) + + # --- g2q (satellite → drone): invert ground-truth --- + n_gallery = gallery.size(0) + valid_q_per_sat: list[set[int]] = [set() for _ in range(n_gallery)] + for q_idx, gset in enumerate(valid_idx_per_query): + for g_idx in gset: + valid_q_per_sat[g_idx].add(q_idx) + + sorted_idx_g2q = sim.t().argsort(dim=1, descending=True) # [N_sat, n_query] + n_scored_g2q = sum(1 for s in valid_q_per_sat if s) + + for k in k_values: + hits_g2q = 0 + for i in range(n_gallery): + valid = valid_q_per_sat[i] + if not valid: + continue + top_k = set(sorted_idx_g2q[i, :k].tolist()) + if valid & top_k: + hits_g2q += 1 + metrics[f"r@{k}_g2q"] = hits_g2q / max(n_scored_g2q, 1) + + mrr_sum_g2q = 0.0 + for i in range(n_gallery): + valid = valid_q_per_sat[i] + if not valid: + continue + for rank, qidx in enumerate(sorted_idx_g2q[i].tolist()): + if qidx in valid: + mrr_sum_g2q += 1.0 / (rank + 1) + break + metrics["ap_g2q"] = mrr_sum_g2q / max(n_scored_g2q, 1) + metrics["n_query"] = float(n_query) - metrics["n_gallery"] = float(gallery.size(0)) + metrics["n_gallery"] = float(n_gallery) + metrics["n_scored_g2q"] = float(n_scored_g2q) metrics["gate_q"] = model.fusion_query.gate_value metrics["gate_g"] = model.fusion_gallery.gate_value @@ -953,17 +987,26 @@ def train(cfg: TrainConfigGTAUAV) -> None: train_row["r@5_q2g"] = train_recall.get("r@5_q2g", 0.0) train_row["r@10_q2g"] = train_recall.get("r@10_q2g", 0.0) train_row["ap_q2g"] = train_recall.get("ap_q2g", 0.0) + train_row["r@1_g2q"] = train_recall.get("r@1_g2q", 0.0) + train_row["r@5_g2q"] = train_recall.get("r@5_g2q", 0.0) + train_row["r@10_g2q"] = train_recall.get("r@10_g2q", 0.0) + train_row["ap_g2q"] = train_recall.get("ap_g2q", 0.0) csv_logger.log_train(epoch, train_row, optimizer.param_groups[0]["lr"], elapsed) generate_plots(csv_logger.log_dir) if train_recall: LOGGER.info( - "train-recall epoch=%d R@1=%.4f R@5=%.4f R@10=%.4f AP=%.4f loss=%.4f", + "train-recall epoch=%d q2g R@1=%.4f R@5=%.4f R@10=%.4f AP=%.4f " + "g2q R@1=%.4f R@5=%.4f R@10=%.4f AP=%.4f loss=%.4f", epoch, train_recall.get("r@1_q2g", 0.0), train_recall.get("r@5_q2g", 0.0), train_recall.get("r@10_q2g", 0.0), train_recall.get("ap_q2g", 0.0), + train_recall.get("r@1_g2q", 0.0), + train_recall.get("r@5_g2q", 0.0), + train_recall.get("r@10_g2q", 0.0), + train_recall.get("ap_g2q", 0.0), train_recall.get("loss", 0.0), ) @@ -985,12 +1028,17 @@ def train(cfg: TrainConfigGTAUAV) -> None: tracker.log_scalar("val/best_r@1_q2g", best_r1, step=global_step) LOGGER.info( - "val epoch=%d R@1=%.4f R@5=%.4f R@10=%.4f AP=%.4f loss=%.4f gate_q=%.4f", + "val epoch=%d q2g R@1=%.4f R@5=%.4f R@10=%.4f AP=%.4f " + "g2q R@1=%.4f R@5=%.4f R@10=%.4f AP=%.4f loss=%.4f gate_q=%.4f", epoch, val_metrics.get("r@1_q2g", 0.0), val_metrics.get("r@5_q2g", 0.0), val_metrics.get("r@10_q2g", 0.0), val_metrics.get("ap_q2g", 0.0), + val_metrics.get("r@1_g2q", 0.0), + val_metrics.get("r@5_g2q", 0.0), + val_metrics.get("r@10_g2q", 0.0), + val_metrics.get("ap_g2q", 0.0), val_metrics.get("loss", 0.0), val_metrics.get("gate_q", 1.0), ) @@ -1061,6 +1109,11 @@ def train(cfg: TrainConfigGTAUAV) -> None: "final_r@1_q2g": final_metrics.get("r@1_q2g", 0.0), "final_r@5_q2g": final_metrics.get("r@5_q2g", 0.0), "final_r@10_q2g": final_metrics.get("r@10_q2g", 0.0), + "final_ap_q2g": final_metrics.get("ap_q2g", 0.0), + "final_r@1_g2q": final_metrics.get("r@1_g2q", 0.0), + "final_r@5_g2q": final_metrics.get("r@5_g2q", 0.0), + "final_r@10_g2q": final_metrics.get("r@10_g2q", 0.0), + "final_ap_g2q": final_metrics.get("ap_g2q", 0.0), "final_gate_q": final_metrics.get("gate_q", 1.0), "final_gate_g": final_metrics.get("gate_g", 1.0), }) @@ -1073,10 +1126,16 @@ def train(cfg: TrainConfigGTAUAV) -> None: LOGGER.info("Training complete. Report: %s", report_path) LOGGER.info( - "Final — R@1=%.4f R@5=%.4f R@10=%.4f gate_q=%.4f gate_g=%.4f", + "Final — q2g R@1=%.4f R@5=%.4f R@10=%.4f AP=%.4f " + "g2q R@1=%.4f R@5=%.4f R@10=%.4f AP=%.4f gate_q=%.4f gate_g=%.4f", final_metrics.get("r@1_q2g", 0.0), final_metrics.get("r@5_q2g", 0.0), final_metrics.get("r@10_q2g", 0.0), + final_metrics.get("ap_q2g", 0.0), + final_metrics.get("r@1_g2q", 0.0), + final_metrics.get("r@5_g2q", 0.0), + final_metrics.get("r@10_g2q", 0.0), + final_metrics.get("ap_g2q", 0.0), final_metrics.get("gate_q", 1.0), final_metrics.get("gate_g", 1.0), )