Rewrite: GatedFusion architecture + UAV-GeoLoc dataset

Architecture v2:
- Query branch: drone + text -> GatedFusion -> proj -> query_emb
- Gallery branch: satellite -> proj -> gallery_emb
- Single InfoNCE loss (asymmetric 0.6/0.4)
- GatedFusion: learnable gated addition (sigma(alpha)*img + (1-sigma(alpha))*text)
- Baseline mode: gate=1.0 (text ignored)

Dataset:
- UAV-GeoLoc loader with template captions from path metadata
- 27 terrain types with predefined features
- Random positive crop sampling per epoch

Configs: balanced (gate=0.7), baseline (no text), text_heavy (gate=0.3)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pikaliov
2026-04-17 17:13:00 +03:00
parent 2ce4017ebd
commit abb3337f8d
12 changed files with 1077 additions and 781 deletions

View File

@@ -1,9 +1,6 @@
from __future__ import annotations
"""Compare baseline vs full-caption runs and compute Delta R@1 report.
Reads eval reports produced by src.eval.evaluate.run_evaluation_from_checkpoint
and produces a markdown + JSON summary.
"""Compare baseline (no text) vs caption-fused runs. Compute Delta R@1 report.
Usage:
python -m scripts.compare_runs \
@@ -18,10 +15,8 @@ from pathlib import Path
_DIRECTIONS = (
"drone_to_sat",
"sat_to_drone",
"text_to_sat",
"text_to_drone",
"query_to_gallery",
"gallery_to_query",
)
_KS = (1, 5, 10)
@@ -33,73 +28,70 @@ def _load_metrics(report_path: Path) -> dict[str, float]:
def _format_row(name: str, baseline: dict[str, float], full: dict[str, float]) -> str:
"""Render one markdown row for a direction across R@1, R@5, R@10."""
cells = [name]
for k in _KS:
key = f"r@{k}_{name}"
b = baseline.get(key, float("nan"))
f_ = full.get(key, float("nan"))
delta = f_ - b if (b == b and f_ == f_) else float("nan") # NaN-safe
cells.append(f"{b:.4f} {f_:.4f} (Δ {delta:+.4f})")
delta = f_ - b if (b == b and f_ == f_) else float("nan")
cells.append(f"{b:.4f} -> {f_:.4f} (D {delta:+.4f})")
return "| " + " | ".join(cells) + " |"
def _interpret_delta(delta: float) -> str:
"""Human-readable caption-quality verdict."""
if delta >= 0.03:
return "PASS captions informative (Δ R@1 +3%)"
return "PASS -- captions informative (D R@1 >= +3%)"
if delta >= 0.01:
return "⚠️ MARGINAL consider VLM refinement (+1% ≤ Δ < +3%)"
return "MARGINAL -- consider VLM refinement (+1% <= D < +3%)"
if delta >= 0:
return "WEAK captions add little signal (< +1%)"
return "❌❌ HARMFUL captions confuse model (Δ < 0)"
return "WEAK -- captions add little signal (< +1%)"
return "HARMFUL -- captions confuse model (D < 0)"
def build_comparison_markdown(
baseline: dict[str, float],
full: dict[str, float],
) -> str:
"""Compose markdown comparison report."""
lines: list[str] = ["# Caption Quality Test: Comparison Report", ""]
# Headline Δ R@1 on primary direction.
primary = "drone_to_sat"
primary = "query_to_gallery"
primary_key = f"r@1_{primary}"
primary_delta = full.get(primary_key, 0.0) - baseline.get(primary_key, 0.0)
verdict = _interpret_delta(primary_delta)
lines.append(f"## Primary metric: Δ R@1 ({primary}) = {primary_delta:+.4f}")
lines.append(f"## Primary metric: D R@1 (query->gallery) = {primary_delta:+.4f}")
lines.append("")
lines.append(f"**Verdict:** {verdict}")
lines.append("")
# Full table.
lines.append("## All directions × K")
# Gate value.
gate = full.get("gate", None)
if gate is not None:
lines.append(f"**Fusion gate:** {gate:.4f} (1.0 = text ignored, 0.0 = image ignored)")
lines.append("")
lines.append("## All directions x K")
lines.append("")
header = "| Direction | R@1 base full | R@5 base full | R@10 base full |"
header = "| Direction | R@1 base -> full | R@5 base -> full | R@10 base -> full |"
sep = "|---|---|---|---|"
lines.extend([header, sep])
for direction in _DIRECTIONS:
row = _format_row(direction, baseline, full)
lines.append(row)
lines.append(_format_row(direction, baseline, full))
lines.append("")
# Decision rule recap.
lines.append("## Decision rule")
lines.append("")
lines.append("- Δ R@1 +3% captions pass, proceed to World-UAV generation")
lines.append("- +1% ≤ Δ R@1 < +3% add VLM refinement, re-run")
lines.append("- Δ R@1 < +1% redesign caption pipeline")
lines.append("- Δ R@1 < 0 critical bug, investigate caption/image alignment")
lines.append("- D R@1 >= +3% -> captions pass, proceed to production")
lines.append("- +1% <= D R@1 < +3% -> add VLM refinement, re-run")
lines.append("- D R@1 < +1% -> redesign caption pipeline")
lines.append("- D R@1 < 0 -> critical bug, investigate")
lines.append("")
return "\n".join(lines)
def main() -> None:
parser = argparse.ArgumentParser(
description="Compare baseline vs full-caption runs."
)
parser = argparse.ArgumentParser(description="Compare baseline vs caption runs.")
parser.add_argument("--baseline_report", type=Path, required=True)
parser.add_argument("--full_report", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
@@ -114,7 +106,6 @@ def main() -> None:
with args.output.open("w", encoding="utf-8") as f:
f.write(md)
# Also write machine-readable summary.
summary = {
"baseline_metrics": baseline,
"full_metrics": full,