From c9083c89ccc43d105620105e1b5dd9a60c61d485 Mon Sep 17 00:00:00 2001 From: pikaliov Date: Thu, 7 May 2026 10:32:30 +0300 Subject: [PATCH] claude_refactor_v3: Test real trainer loading before training loop + Add log.md --- belka_refactor_07_05_log.md | 183 ++++++++++++++++++++++++++++++++++++ src/main.py | 2 +- src/training/trainer_new.py | 2 +- tests/conftest.py | 137 --------------------------- 4 files changed, 185 insertions(+), 139 deletions(-) create mode 100644 belka_refactor_07_05_log.md diff --git a/belka_refactor_07_05_log.md b/belka_refactor_07_05_log.md new file mode 100644 index 0000000..84e20d4 --- /dev/null +++ b/belka_refactor_07_05_log.md @@ -0,0 +1,183 @@ +# Шаг 4b — Что изменилось + +--- + +## 1. Новая точка входа — `src/main.py` + добавлена debug-конфигурация `Main entry point` в `launch.json` + +**Пример запуска отладки:** +```bash +python src/training/train_gtauav.py --config conf/gtauav_balanced.gin +``` + +## 2. Тесты + +Добавлены `tests\conftest.py & test_trainer.py` + +### Тест загрузки конфигураций в трейнер +Файл `test_trainer.py` + +**Пример запуска отладки:** +```bash +python -m pytest tests/test_trainer.py +``` + +**Лог:** + +```bash +=========================================== test session starts =========================================== +platform linux -- Python 3.10.19, pytest-9.0.3, pluggy-1.6.0 +rootdir: /home/servml/Документы/code/Yaroslav/caption-test +collected 21 items + +tests/test_trainer.py . ✅ _SUPPORTED_BACKBONES must be a frozenset (immutable, hashable). +. ✅ Exactly dinov3 and stripnet are supported in the current refactor. +. ✅ ModelsConfig union mirrors _SUPPORTED_BACKBONES (dinov3 | stripnet). +. +Supported backbones pass _validate_backbone silently. + ✅ dinov3 +. ✅ stripnet +. +Unsupported backbones (incl. sofia) raise NotImplementedError, not ImportError. + ✅ sofia_v1 +. ✅ sofia_v71 +. ✅ mistral_42b +. ✅ empty +. +Trainer(...) instantiates from every real preset's loaded cfgs. + ✅ gtauav_balanced +. ✅ gtauav_balanced_asym +. ✅ gtauav_baseline +. ✅ gtauav_baseline_asym +. ✅ gtauav_image_heavy +. ✅ gtauav_text_heavy +. ✅ gtauav_balanced_stripnet +. ✅ gtauav_balanced_stripnet_unfrozen +. ✅ gtauav_baseline_stripnet +. ✅ gtauav_baseline_stripnet_unfrozen +. ✅ All runtime fields are None / 0 / [] before .train() is called. +. ✅ Trainer.train() takes only `self` — main.py calls trainer.train(). + + [100%] + +=========================================== 21 passed in 2.19s =========================================== +``` + +## 3. Декомпозиция вызовов в `trainer_new.py` + +**Теперь вызовы выглядят такой цепочкой:** +```python +def train(self) -> None: + """Full pipeline: setup → build → train → evaluate → cleanup.""" + self._validate_backbone() + clear_vram() + set_seed(self.pipeline_cfg.seed) + self._setup_output_dir() + self._setup_tracker() + self._build_model() + self._configure_gradient_checkpointing() + self._log_model_summary() + self._build_loss() + self._build_neg_bank() + self._build_data_loaders() + self._build_optimizer_and_scheduler() + self._restore_from_resume() + self._setup_profiler() + + try: + self._train_loop() + self._final_evaluation() + finally: + self._cleanup() +``` + +**Лог:** + +### Тест вызовов до блока `try: self._train_loop()` +✅ Загрузка весов +✅ Сборка пайплайна +✅ Загрузка моделей +✅ Загрузка данных + +```bash +2026-05-07 09:40:53 caption_test.trainer INFO ⚙️ Validate backbone +2026-05-07 09:40:53 caption_test.trainer INFO ⚙️ Setup out dir +2026-05-07 09:40:53 caption_test.trainer INFO ⚙️ Setup tracker... +2026-05-07 09:40:53 caption_test.trackers WARNING tensorboard not installed, skipping TB tracking +2026-05-07 09:41:24 caption_test.trainer INFO ⚙️ Build loss... +2026-05-07 09:41:52 caption_test.trainer INFO Building model — with text (L1/L2/L3), shared DINOv3 WEB +2026-05-07 09:42:24 caption_test.trainer INFO ⚙️ Build DINOv3 model... +2026-05-07 09:50:48 caption_test.model INFO 🧊 Loading DINOv3 from dinov3-vitl16-pretrain-lvd1689m.pth +2026-05-07 09:50:48 caption_test.model INFO 🧊 Loading DINOv3 from dinov3-vitl16-pretrain-lvd1689m.pth +2026-05-07 09:54:48 caption_test.model INFO 🧊 DINOv3 loaded: 303,129,600 params +2026-05-07 09:54:48 caption_test.model INFO 🧊 DINOv3 loaded: 303,129,600 params +2026-05-07 09:55:03 caption_test.adapters INFO 🔧 MONA injected: 24 adapters (blocks 12-23 of 24), 3,502,080 trainable params (bottleneck=64) +2026-05-07 09:55:03 caption_test.adapters INFO 🔧 MONA injected: 24 adapters (blocks 12-23 of 24), 3,502,080 trainable params (bottleneck=64) +2026-05-07 09:55:05 caption_test.model INFO Shared encoder mode: single DINOv3 WEB for drone + satellite +2026-05-07 09:55:05 caption_test.model INFO Shared encoder mode: single DINOv3 WEB for drone + satellite +2026-05-07 09:55:38 caption_test.dgtrs INFO 📝 Loading DGTRS-CLIP text encoder from DGTRS-CLIP-ViT-L-14.pt +2026-05-07 09:55:38 caption_test.dgtrs INFO 📝 Loading DGTRS-CLIP text encoder from DGTRS-CLIP-ViT-L-14.pt +2026-05-07 09:55:39 caption_test.dgtrs INFO 📝 DGTRS text encoder loaded: 123,972,096 params, context=248 tokens +2026-05-07 09:55:39 caption_test.dgtrs INFO 📝 DGTRS text encoder loaded: 123,972,096 params, context=248 tokens +2026-05-07 10:03:01 caption_test.adapters INFO 🔧 LoRA injected: 12 blocks, rank=4, 147,456 trainable params +2026-05-07 10:03:01 caption_test.adapters INFO 🔧 LoRA injected: 12 blocks, rank=4, 147,456 trainable params +2026-05-07 10:04:01 caption_test.trainer INFO embed_dim=1024 +2026-05-07 10:04:22 caption_test.trainer INFO ⚙️ Configure gradient checkpointing... +2026-05-07 10:04:42 caption_test.trainer INFO ✅ Gradient checkpointing enabled (DINOv3 + DGTRS) +2026-05-07 10:04:51 caption_test.trainer INFO trainable=7,059,458 (1.63%) total=434,161,154 +2026-05-07 10:04:51 caption_test.profiler INFO torchinfo not installed, using basic parameter count +2026-05-07 10:04:51 caption_test.profiler INFO Model summary: +Total parameters: 434,161,154 +Trainable parameters: 7,059,458 (1.63%) + [trainable] image_encoder.layer.12.mona_attn.gamma: [1024] (1,024) + ... (Вывод списка параметров) + ... +2026-05-07 10:05:00 caption_test.trainer INFO ⚙️ Build loss... +2026-05-07 10:06:06 caption_test.trainer INFO Loss: SymmetricInfoNCE Temperature: learnable (init=0.070) q2g=0.60 g2q=0.40 +2026-05-07 10:06:07 caption_test.trainer INFO ⚙️ Build negative bank... +2026-05-07 10:06:13 caption_test.trainer INFO ⚙️ Build dataloaders... +2026-05-07 10:06:13 caption_test.gtauav_dataset INFO 🔻 Filter loaded: 10905 excluded images +2026-05-07 10:06:13 caption_test.gtauav_dataset INFO 🔻 Filter loaded: 10905 excluded images +2026-05-07 10:06:13 caption_test.gtauav_dataset INFO 📐 Altitude index: 33708 drones (from 2 CSV) +2026-05-07 10:06:13 caption_test.gtauav_dataset INFO 📐 Altitude index: 33708 drones (from 2 CSV) +2026-05-07 10:06:13 caption_test.gtauav_dataset INFO 📚 Loading caption index from /home/servml/Документы/datasets/GTA-UAV-LR-captions +2026-05-07 10:06:13 caption_test.gtauav_dataset INFO 📚 Loading caption index from /home/servml/Документы/datasets/GTA-UAV-LR-captions +2026-05-07 10:06:19 caption_test.gtauav_dataset INFO 📚 Caption index: 39957 entries +2026-05-07 10:06:19 caption_test.gtauav_dataset INFO 📚 Caption index: 39957 entries +2026-05-07 10:06:20 caption_test.gtauav_dataset INFO ✅ Loaded 24891 pairs from meta/train_80.json +2026-05-07 10:06:20 caption_test.gtauav_dataset INFO ✅ Loaded 24891 pairs from meta/train_80.json +2026-05-07 10:06:20 caption_test.gtauav_dataset INFO 🔻 Filter loaded: 10905 excluded images +2026-05-07 10:06:20 caption_test.gtauav_dataset INFO 🔻 Filter loaded: 10905 excluded images +2026-05-07 10:06:20 caption_test.gtauav_dataset INFO 📐 Altitude index: 33708 drones (from 2 CSV) +2026-05-07 10:06:20 caption_test.gtauav_dataset INFO 📐 Altitude index: 33708 drones (from 2 CSV) +2026-05-07 10:06:20 caption_test.gtauav_dataset INFO 📚 Loading caption index from /home/servml/Документы/datasets/GTA-UAV-LR-captions +2026-05-07 10:06:20 caption_test.gtauav_dataset INFO 📚 Loading caption index from /home/servml/Документы/datasets/GTA-UAV-LR-captions +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO 📚 Caption index: 39957 entries +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO 📚 Caption index: 39957 entries +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO ✅ Loaded 6252 pairs from meta/test_20.json +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO ✅ Loaded 6252 pairs from meta/test_20.json +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO 🔻 Filter loaded: 10905 excluded images +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO 🔻 Filter loaded: 10905 excluded images +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO 📐 Altitude index: 33708 drones (from 2 CSV) +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO 📐 Altitude index: 33708 drones (from 2 CSV) +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO 📚 Loading caption index from /home/servml/Документы/datasets/GTA-UAV-LR-captions +2026-05-07 10:06:22 caption_test.gtauav_dataset INFO 📚 Loading caption index from /home/servml/Документы/datasets/GTA-UAV-LR-captions +2026-05-07 10:06:24 caption_test.gtauav_dataset INFO 📚 Caption index: 39957 entries +2026-05-07 10:06:24 caption_test.gtauav_dataset INFO 📚 Caption index: 39957 entries +2026-05-07 10:06:25 caption_test.gtauav_dataset INFO ✅ Loaded 24891 pairs from meta/train_80.json +2026-05-07 10:06:25 caption_test.gtauav_dataset INFO ✅ Loaded 24891 pairs from meta/train_80.json +2026-05-07 10:06:25 caption_test.trainer INFO Sampler: MutuallyExclusive — no false negatives within a batch +2026-05-07 10:06:25 caption_test.trainer INFO train=24891 test=6252 batch=8 accum=8 effective_batch=64 +2026-05-07 10:06:46 caption_test.trainer INFO ⚙️ Build optimizer & scheduler... +2026-05-07 10:06:46 caption_test.trainer INFO Optimizer: AdamW LR: proj=1e-04 text=1e-05 warmup=2 epochs +2026-05-07 10:07:33 caption_test.trainer INFO ⚙️ Restore from resume... +2026-05-07 10:07:37 caption_test.trainer INFO ⚙️ Setup profiler... +``` + +`src/main.py`: +- Читает имя пресета из `sys.argv[1]` (один позиционный аргумент) +- Резолвит корень проекта через `get_proj_dir()` (поиск по маркерам `pyproject.toml`/`.git`/`in/`) +- Формирует `path2cfg = f"{proj_dir}in/config_files/"` буквально по REQUIREMENTS_GIN_STYLE.md §5 +- Вызывает `load_all_configs(path2cfg, preset_name)` — двухпроходная загрузка из `_common`-файлов и пресет-директории +- Передаёт 6 объектов конфига в `train(...)` именованными аргументами + + diff --git a/src/main.py b/src/main.py index dfe8bcc..d6a5286 100644 --- a/src/main.py +++ b/src/main.py @@ -4,7 +4,7 @@ Usage: python -m src.main gtauav_balanced python -m src.main gtauav_balanced_stripnet -! # NOTE: Add debug config +! # NOTE: Added debug config """ from __future__ import annotations diff --git a/src/training/trainer_new.py b/src/training/trainer_new.py index 61dee78..6304b03 100644 --- a/src/training/trainer_new.py +++ b/src/training/trainer_new.py @@ -230,7 +230,7 @@ class Trainer: def train(self) -> None: """Full pipeline: setup → build → train → evaluate → cleanup.""" self._validate_backbone() - #! clear_vram() + clear_vram() set_seed(self.pipeline_cfg.seed) self._setup_output_dir() self._setup_tracker() diff --git a/tests/conftest.py b/tests/conftest.py index 4fa4f25..422348c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,140 +1,3 @@ -# """Shared pytest fixtures + reporter hooks for caption-test test suite. - -# Provides: -# - proj_dir, path2cfg: real repository paths for integration-style tests -# that load actual gin presets from in/config_files/. -# - clear_gin: autouse fixture that wipes gin global state before every test -# (gin keeps bindings in module-level singleton; tests must not leak). - -# Reporter hooks print "✅/❌ " next to each test's PASSED/FAILED -# line in -v mode. - -# Preset name lists live in a separate module (`tests/_presets.py`) so test -# files can import them with a plain `import _presets` — no relative imports, -# no need for tests/ to be a package. -# """ - -# from __future__ import annotations - -# from pathlib import Path - -# import gin -# import pytest - - -# DINOV3_PRESETS = ( -# "gtauav_balanced", -# "gtauav_balanced_asym", -# "gtauav_baseline", -# "gtauav_baseline_asym", -# "gtauav_image_heavy", -# "gtauav_text_heavy", -# ) - -# STRIPNET_PRESETS = ( -# "gtauav_balanced_stripnet", -# "gtauav_balanced_stripnet_unfrozen", -# "gtauav_baseline_stripnet", -# "gtauav_baseline_stripnet_unfrozen", -# ) - -# ALL_TRAINING_PRESETS = DINOV3_PRESETS + STRIPNET_PRESETS - -# # --- gin hygiene ----------------------------------------------------------- - - -# @pytest.fixture(autouse=True) -# def clear_gin(): -# """Wipe gin's global binding state before AND after each test. - -# Gin keeps bindings in module-level singletons; without this fixture, a -# test that loads a config (or even just calls gin.parse_config_file in a -# helper) leaks bindings into the next test, leading to flaky failures -# like 'Unknown configurable' or wrong field values. -# """ -# gin.clear_config() -# yield -# gin.clear_config() - - -# # --- real-repo paths ------------------------------------------------------- - - -# @pytest.fixture -# def proj_dir() -> Path: -# """Path to repository root (the directory that contains src/, tests/, in/).""" -# return Path(__file__).resolve().parent.parent - - -# @pytest.fixture -# def path2cfg(proj_dir: Path) -> str: -# """Trailing-slashed path to in/config_files/, matching `src/main.py`. - -# Per REQUIREMENTS_GIN_STYLE.md §5, src/main.py builds this path as -# `f"{proj_dir}in/config_files/"`. Tests that exercise the real repo -# layout should use this fixture verbatim instead of constructing it -# independently. -# """ -# return f"{proj_dir}/in/config_files/" - - -# # --- reporter hooks -------------------------------------------------------- - - -# def _docstring_summary(item: pytest.Item) -> str | None: -# """Return the first non-empty line of a test's docstring, or None.""" -# func = getattr(item, "function", None) or getattr(item, "obj", None) -# if func is None or not getattr(func, "__doc__", None): -# return None -# for line in func.__doc__.strip().splitlines(): -# stripped = line.strip() -# if stripped: -# return stripped -# return None - - -# # Cache nodeid → docstring summary, populated at collection time so the -# # logreport hook can look them up without re-introspecting the test function. -# _LAST_SEEN_SUMMARY: dict[str, str] = {} - - -# def pytest_collection_modifyitems( -# config: pytest.Config, -# items: list[pytest.Item], -# ) -> None: -# """Cache each item's docstring summary for later use by the status hook.""" -# for item in items: -# summary = _docstring_summary(item) -# if summary: -# _LAST_SEEN_SUMMARY[item.nodeid] = summary - - -# def pytest_runtest_logreport(report: pytest.TestReport) -> None: -# """Print '✅/❌/⏭️ ' after each test's `call` phase finishes. - -# Pytest emits 3 reports per test (setup → call → teardown). We hook the -# `call` phase — the one where pass/fail is actually decided — and write -# the icon + docstring summary to stdout, where it appears next to -# pytest's own PASSED/FAILED line. -# """ -# if report.when != "call": -# return - -# summary = _LAST_SEEN_SUMMARY.get(report.nodeid, "(no docstring)") -# if report.passed: -# icon = "✅" -# elif report.failed: -# icon = "❌" -# else: -# icon = "⏭️" -# print(f" {icon} {summary}") - - - - - - - """Shared pytest fixtures + reporter hooks for caption-test test suite. Provides: