From 531a65c2f05f491b78ce32044aa8c104f868196d Mon Sep 17 00:00:00 2001 From: pikaliov Date: Wed, 8 Jul 2026 11:31:51 +0300 Subject: [PATCH] fix --- check_coverage.py | 66 +++++++++++++++++++++++++++++++++++----------- collect_results.py | 45 ++++++++++++++++++++++--------- 2 files changed, 83 insertions(+), 28 deletions(-) diff --git a/check_coverage.py b/check_coverage.py index 011f129..90a879d 100644 --- a/check_coverage.py +++ b/check_coverage.py @@ -9,12 +9,13 @@ combine_text_levels), поэтому «есть текст» здесь озна видит модель: непустой текст для выбранных text_levels, а не заглушку "No description available.". -Пример: - python check_coverage.py \ - --data_root /media/servml/SSD_2_2TB/datasets/cvgl_datasets/GTA-UAV-LR \ - --captions_v1 ".../GTA-UAV-LR-captions_ v1" \ - --captions_v2 ".../GTA-UAV-LR-captions_v2" \ - --text_levels level1 level2 +Запуск: + python3 check_coverage.py \ + --data_root "/media/servml/SSD_2_2TB/datasets/cvgl_datasets/GTA-UAV-LR" \ + --captions_v1 "/media/servml/SSD_2_2TB/datasets/cvgl_datasets/GTA-UAV-LR-captions_ v1" \ + --captions_v2 "/media/servml/SSD_2_2TB/datasets/cvgl_datasets/GTA-UAV-LR-captions_v2" \ + --text_levels level1 level2 level3 + """ from __future__ import annotations @@ -39,8 +40,10 @@ def _has_real_text( desc = descriptions.get(name) if not desc: return False + # combine_text_levels возвращает заглушку, когда все уровни пустые → + # это и есть «текста фактически нет». combined = combine_text_levels(desc, text_levels) - return combined.strip() != "" and combined != "No description available." + return combined != "No description available." def _covered_set( @@ -87,33 +90,64 @@ def _report_side( ) -def main(args: argparse.Namespace) -> None: - data_root = Path(args.data_root) - text_levels = list(args.text_levels) +def _report_split( + data_root: Path, + meta_name: str, + desc_by_view: dict[str, tuple[dict, dict]], + text_levels: list[str], +) -> None: + """Покрытие v1/v2 для одного split-файла (train или test).""" + meta_path = data_root / meta_name + if not meta_path.exists(): + print(f"\n⚠️ Пропущен (нет файла): {meta_path}") + return - with open(data_root / args.test_meta) as f: + with open(meta_path) as f: pairs_meta = json.load(f) - # Дрон-запросы: уникальные имена картинок дрона в тесте. + # Дрон-запросы: уникальные имена картинок дрона в split. drone_names = sorted({e["drone_img_name"] for e in pairs_meta}) # Спутниковая галерея: те же уникальные тайлы, что и в eval. gallery_names, _ = collect_gallery_names(pairs_meta) - print("=" * 64) - print(f"COVERAGE CHECK | text_levels={text_levels} | {args.test_meta}") + print("\n" + "=" * 64) + print(f"SPLIT: {meta_name} ({len(pairs_meta)} пар)") print("=" * 64) for view, names in (("drone", drone_names), ("satellite", gallery_names)): - desc_v1 = load_text_descriptions(args.captions_v1, view_type=view) - desc_v2 = load_text_descriptions(args.captions_v2, view_type=view) + desc_v1, desc_v2 = desc_by_view[view] _report_side( f"{view.upper()} {'queries' if view == 'drone' else 'gallery'}", names, desc_v1, desc_v2, text_levels, ) + +def main(args: argparse.Namespace) -> None: + data_root = Path(args.data_root) + text_levels = list(args.text_levels) + + print("=" * 64) + print(f"COVERAGE CHECK | text_levels={text_levels}") + print("=" * 64) + + # Описания не зависят от split — грузим один раз на каждую сторону. + desc_by_view = { + view: ( + load_text_descriptions(args.captions_v1, view_type=view), + load_text_descriptions(args.captions_v2, view_type=view), + ) + for view in ("drone", "satellite") + } + + # Покрытие важно и на train (что модель УЧИТ), и на test (что видит eval). + for meta_name in (args.train_meta, args.test_meta): + _report_split(data_root, meta_name, desc_by_view, text_levels) + + def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description="Сравнить покрытие captions v1 vs v2") p.add_argument("--data_root", type=str, required=True) + p.add_argument("--train_meta", type=str, default="cross-area-drone2sate-train.json") p.add_argument("--test_meta", type=str, default="cross-area-drone2sate-test.json") p.add_argument("--captions_v1", type=str, required=True) p.add_argument("--captions_v2", type=str, required=True) diff --git a/collect_results.py b/collect_results.py index cd5289e..9c0c459 100644 --- a/collect_results.py +++ b/collect_results.py @@ -42,19 +42,33 @@ def load_experiment(exp_dir: Path) -> dict | None: if not history: return None + # primary метрика — q2g R@1 (drone→satellite), см. протокол §6.2; + # eval_recall@* дублируют q2g, eval_mAP дублирует q2g_mAP. best = max(history, key=lambda r: r.get("eval_recall@1", 0)) latest = history[-1] return { "dir": exp_dir.name, + "seed": config.get("seed", "?"), "text_levels": " + ".join(config.get("text_levels", ["?"])), "epochs_done": latest["epoch"], "epochs_total": config.get("epochs", "?"), "best_epoch": best["epoch"], + # q2g (primary); split — только official cross-area (§5.3), + # same-area прогонов нет, поэтому cross−same gap не считается. "best_R@1": best.get("eval_recall@1", 0), "best_R@5": best.get("eval_recall@5", 0), "best_R@10": best.get("eval_recall@10", 0), - "best_AP": best.get("eval_AP", 0), + "best_R@1%": best.get("eval_q2g_recall@1%", 0), + "best_mAP": best.get("eval_mAP", 0), # настоящий AP, не MRR + "best_median_rank": best.get("eval_q2g_median_rank", 0), + "best_mean_rank": best.get("eval_q2g_mean_rank", 0), + # g2q (satellite→drone), та же best-эпоха + "best_g2q_R@1": best.get("eval_g2q_recall@1", 0), + "best_g2q_R@5": best.get("eval_g2q_recall@5", 0), + "best_g2q_R@10": best.get("eval_g2q_recall@10", 0), + "best_g2q_R@1%": best.get("eval_g2q_recall@1%", 0), + "best_g2q_mAP": best.get("eval_g2q_mAP", 0), "latest_loss": latest.get("train_loss", 0), "latest_R@1": latest.get("eval_recall@1", 0), "avg_epoch_time": sum(r.get("elapsed_s", 0) for r in history) / len(history), @@ -63,14 +77,17 @@ def load_experiment(exp_dir: Path) -> dict | None: def collect_version(version: str) -> list[dict]: - """Собрать результаты одной версии (v1 или v2).""" + """Собрать результаты одной версии (v1 или v2). + + Структура: outputs////history.json. + """ version_dir = OUTPUTS_DIR / version if not version_dir.exists(): print(f"⚠️ Папка не найдена: {version_dir}") return [] results = [] - for exp_dir in sorted(version_dir.iterdir()): + for exp_dir in sorted(version_dir.glob("*/*")): if exp_dir.is_dir() and (exp_dir / "history.json").exists(): data = load_experiment(exp_dir) if data: @@ -91,14 +108,14 @@ def print_table(results: list[dict], version: str) -> None: results.sort(key=lambda r: -r["best_R@1"]) header = ( - f"{'Levels':<24} {'Prog':<8} " - f"{'BestEp':>6} {'R@1':>7} {'R@5':>7} {'R@10':>7} " - f"{'AP':>7} {'Loss':>8} {'Time':>6}" + f"{'Levels':<24} {'Seed':>5} {'Prog':<8} {'BestEp':>6} " + f"{'R@1':>7} {'R@5':>7} {'R@10':>7} {'R@1%':>7} {'mAP':>7} " + f"{'medR':>6} {'g2q@1':>7} {'Loss':>8} {'Time':>6}" ) sep = "─" * len(header) print(sep) - print(f" {version.upper()} Results") + print(f" {version.upper()} Results (q2g = primary; g2q@1 = satellite→drone)") print(sep) print(header) print(sep) @@ -106,9 +123,10 @@ def print_table(results: list[dict], version: str) -> None: for r in results: prog = f"{r['epochs_done']}/{r['epochs_total']}" print( - f"{r['text_levels']:<24} {prog:<8} " - f"{r['best_epoch']:>6} {r['best_R@1']:>7.4f} {r['best_R@5']:>7.4f} " - f"{r['best_R@10']:>7.4f} {r['best_AP']:>7.4f} " + f"{r['text_levels']:<24} {str(r['seed']):>5} {prog:<8} {r['best_epoch']:>6} " + f"{r['best_R@1']:>7.4f} {r['best_R@5']:>7.4f} {r['best_R@10']:>7.4f} " + f"{r['best_R@1%']:>7.4f} {r['best_mAP']:>7.4f} {r['best_median_rank']:>6.0f} " + f"{r['best_g2q_R@1']:>7.4f} " f"{r['latest_loss']:>8.4f} {r['avg_epoch_time']:>5.0f}s" ) @@ -123,8 +141,11 @@ def print_table(results: list[dict], version: str) -> None: def save_csv(results: list[dict], version: str) -> None: fields = [ - "version", "text_levels", "epochs_done", "epochs_total", - "best_epoch", "best_R@1", "best_R@5", "best_R@10", "best_AP", + "version", "seed", "text_levels", "epochs_done", "epochs_total", + "best_epoch", + "best_R@1", "best_R@5", "best_R@10", "best_R@1%", "best_mAP", + "best_median_rank", "best_mean_rank", + "best_g2q_R@1", "best_g2q_R@5", "best_g2q_R@10", "best_g2q_R@1%", "best_g2q_mAP", "latest_loss", "latest_R@1", "avg_epoch_time", "dir", ] path = RESULTS_DIR / f"results_{version}.csv"