forked from Pikaliov/fuze_task
fuse_proj: Initial operational package for 3 researchers (Pavlenko/Blizno/Moroz)
Multimodal fusion research on StripNet+GTA-UAV proxy: - 3 independent fusion tracks: condition-aware (A), token/bottleneck (B), role-aware (C) - Shared interfaces, protocol, dataset audit, baseline benchmarks - Canonical version-chain references to vault (SPEC, ANALYSIS, TRIAGE) - Personalized task plans and decision tables for each researcher - 3 generated DOCX task assignment files with milestones and DoD checklist - Full modality dropout diagnostics and missing-modality robustness requirements - Data contract, benchmark registry, experiment tracking infrastructure Operational documents: - docs/00_project/: MERIDIAN context, protocol, repository reuse guide, experiment specification - docs/01_tasks/: Master assignment + 3 individual researcher tracks + joint integration - docs/02_references/: Core literature, version-chain bases, code maps - docs/03_codebase_guides/: Existing code snapshots from vault - scripts/: gen_task_plans.js (DOCX generation), placeholder infrastructure - vendor_reference/: Snapshots of caption_test, depth_edges_annotate, existing SOFIA/SegModel code - reports/, results/, experiments/: Shared output structure for all 3 researchers 3 DOCX files generated from gen_task_plans.js (Times New Roman 14pt, GOST format): - План_заданий_Павленко_БВ.docx (Condition-Aware track, fusion API owner) - План_заданий_Близно_МВ.docx (Token/Bottleneck track, benchmark owner) - План_заданий_Мороз_ЕС.docx (Role-Aware track, data contract owner) Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
Pikaliov
166
vendor_reference/caption_test/src/training/profiling.py
Normal file
166
vendor_reference/caption_test/src/training/profiling.py
Normal file
@@ -0,0 +1,166 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""PyTorch Profiler wrapper for training performance analysis.
|
||||
|
||||
Profiles the first N batches of training to identify bottlenecks
|
||||
in CUDA/CPU execution, memory allocation, and data loading.
|
||||
|
||||
Exports:
|
||||
- Chrome trace (viewable in chrome://tracing)
|
||||
- TensorBoard plugin data (if TB available)
|
||||
- Summary table to console
|
||||
|
||||
Usage:
|
||||
profiler = TrainingProfiler(output_dir, n_warmup=3, n_active=5)
|
||||
for batch_idx, batch in enumerate(loader):
|
||||
with profiler.step_context(batch_idx):
|
||||
# ... training step ...
|
||||
if profiler.is_done(batch_idx):
|
||||
break
|
||||
profiler.export()
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from torch.profiler import ProfilerActivity, profile, schedule, tensorboard_trace_handler
|
||||
|
||||
LOGGER = logging.getLogger("caption_test.profiler")
|
||||
|
||||
|
||||
class TrainingProfiler:
|
||||
"""PyTorch profiler for first N training batches.
|
||||
|
||||
Args:
|
||||
output_dir: Directory for profiler output.
|
||||
n_warmup: Number of warmup steps (not profiled).
|
||||
n_active: Number of steps to actively profile.
|
||||
n_repeat: Number of profiling cycles.
|
||||
record_shapes: Record tensor shapes.
|
||||
profile_memory: Track memory allocation.
|
||||
with_stack: Record Python call stacks.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
output_dir: str | Path,
|
||||
n_warmup: int = 3,
|
||||
n_active: int = 5,
|
||||
n_repeat: int = 1,
|
||||
record_shapes: bool = True,
|
||||
profile_memory: bool = True,
|
||||
with_stack: bool = False,
|
||||
) -> None:
|
||||
self.output_dir = Path(output_dir) / "profiler"
|
||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.n_warmup = n_warmup
|
||||
self.n_active = n_active
|
||||
self.n_repeat = n_repeat
|
||||
self.total_steps = (n_warmup + n_active) * n_repeat
|
||||
|
||||
self._profiler = profile(
|
||||
activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],
|
||||
schedule=schedule(
|
||||
wait=0,
|
||||
warmup=n_warmup,
|
||||
active=n_active,
|
||||
repeat=n_repeat,
|
||||
),
|
||||
on_trace_ready=tensorboard_trace_handler(str(self.output_dir)),
|
||||
record_shapes=record_shapes,
|
||||
profile_memory=profile_memory,
|
||||
with_stack=with_stack,
|
||||
)
|
||||
self._started = False
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start the profiler."""
|
||||
self._profiler.__enter__()
|
||||
self._started = True
|
||||
LOGGER.info(
|
||||
"Profiler started: %d warmup + %d active steps, output: %s",
|
||||
self.n_warmup, self.n_active, self.output_dir,
|
||||
)
|
||||
|
||||
def step(self) -> None:
|
||||
"""Signal end of a profiling step."""
|
||||
if self._started:
|
||||
self._profiler.step()
|
||||
|
||||
def is_done(self, batch_idx: int) -> bool:
|
||||
"""Check if profiling is complete."""
|
||||
return batch_idx >= self.total_steps
|
||||
|
||||
def export(self) -> None:
|
||||
"""Export profiling results and print summary."""
|
||||
if not self._started:
|
||||
return
|
||||
|
||||
self._profiler.__exit__(None, None, None)
|
||||
self._started = False
|
||||
|
||||
# Print key averages summary.
|
||||
summary = self._profiler.key_averages().table(
|
||||
sort_by="cuda_time_total", row_limit=20,
|
||||
)
|
||||
LOGGER.info("Profiler summary (top 20 by CUDA time):\n%s", summary)
|
||||
|
||||
# Export Chrome trace.
|
||||
trace_path = self.output_dir / "chrome_trace.json"
|
||||
self._profiler.export_chrome_trace(str(trace_path))
|
||||
LOGGER.info("Chrome trace exported: %s", trace_path)
|
||||
|
||||
# Memory summary if available.
|
||||
if torch.cuda.is_available():
|
||||
mem_summary = torch.cuda.memory_summary(abbreviated=True)
|
||||
summary_path = self.output_dir / "memory_summary.txt"
|
||||
summary_path.write_text(mem_summary)
|
||||
LOGGER.info("CUDA memory summary: %s", summary_path)
|
||||
|
||||
|
||||
def print_model_summary(model: torch.nn.Module, device: str = "cuda") -> str:
|
||||
"""Print model summary using torchinfo (if available).
|
||||
|
||||
Falls back to a simple parameter count if torchinfo is not installed.
|
||||
|
||||
Returns:
|
||||
Summary string.
|
||||
"""
|
||||
try:
|
||||
from torchinfo import summary as torchinfo_summary
|
||||
info = torchinfo_summary(
|
||||
model,
|
||||
input_data={
|
||||
"drone_img": torch.randn(1, 3, 256, 256, device=device),
|
||||
"sat_img": torch.randn(1, 3, 256, 256, device=device),
|
||||
},
|
||||
col_names=["input_size", "output_size", "num_params", "trainable"],
|
||||
verbose=0,
|
||||
depth=3,
|
||||
)
|
||||
summary_str = str(info)
|
||||
LOGGER.info("Model summary (torchinfo):\n%s", summary_str)
|
||||
return summary_str
|
||||
except ImportError:
|
||||
LOGGER.info("torchinfo not installed, using basic parameter count")
|
||||
except Exception as e:
|
||||
LOGGER.warning("torchinfo failed (%s), using basic parameter count", e)
|
||||
|
||||
# Fallback: simple param count.
|
||||
lines = []
|
||||
total = 0
|
||||
trainable = 0
|
||||
for name, param in model.named_parameters():
|
||||
total += param.numel()
|
||||
if param.requires_grad:
|
||||
trainable += param.numel()
|
||||
lines.append(f" [trainable] {name}: {list(param.shape)} ({param.numel():,})")
|
||||
|
||||
summary_str = (
|
||||
f"Total parameters: {total:,}\n"
|
||||
f"Trainable parameters: {trainable:,} ({100*trainable/max(total,1):.2f}%)\n"
|
||||
+ "\n".join(lines[:30])
|
||||
)
|
||||
LOGGER.info("Model summary:\n%s", summary_str)
|
||||
return summary_str
|
||||
206
vendor_reference/caption_test/src/training/trackers.py
Normal file
206
vendor_reference/caption_test/src/training/trackers.py
Normal file
@@ -0,0 +1,206 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""Unified experiment tracking: W&B + TensorBoard + CSV.
|
||||
|
||||
Auto-detects available backends. Falls back gracefully if wandb/tensorboard
|
||||
are not installed.
|
||||
|
||||
Usage:
|
||||
tracker = ExperimentTracker(output_dir, config_dict, use_wandb=True, use_tb=True)
|
||||
tracker.log_train(epoch, {"loss": 0.5, "lr": 1e-4})
|
||||
tracker.log_val(epoch, {"r@1_q2g": 0.3})
|
||||
tracker.log_gradients(epoch, grad_norms_dict)
|
||||
tracker.log_image(epoch, "gradcam/drone", image_tensor)
|
||||
tracker.close()
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
LOGGER = logging.getLogger("caption_test.trackers")
|
||||
|
||||
|
||||
def _try_import_wandb():
|
||||
try:
|
||||
import wandb
|
||||
return wandb
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
|
||||
def _try_import_tb():
|
||||
try:
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
return SummaryWriter
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
|
||||
class ExperimentTracker:
|
||||
"""Unified tracker dispatching to W&B, TensorBoard, and CSV.
|
||||
|
||||
Args:
|
||||
output_dir: Base output directory.
|
||||
config: Dict of hyperparameters to log.
|
||||
use_wandb: Enable Weights & Biases tracking.
|
||||
use_tb: Enable TensorBoard tracking.
|
||||
wandb_project: W&B project name.
|
||||
wandb_run_name: W&B run name (auto-generated if None).
|
||||
wandb_entity: W&B entity (team/user).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
output_dir: str | Path,
|
||||
config: dict[str, Any] | None = None,
|
||||
use_wandb: bool = False,
|
||||
use_tb: bool = True,
|
||||
wandb_project: str = "caption-test-gtauav",
|
||||
wandb_run_name: str | None = None,
|
||||
wandb_entity: str | None = None,
|
||||
) -> None:
|
||||
self.output_dir = Path(output_dir)
|
||||
self._wandb_run = None
|
||||
self._tb_writer = None
|
||||
|
||||
# W&B init.
|
||||
if use_wandb:
|
||||
wandb = _try_import_wandb()
|
||||
if wandb is not None:
|
||||
self._wandb_run = wandb.init(
|
||||
project=wandb_project,
|
||||
name=wandb_run_name,
|
||||
entity=wandb_entity,
|
||||
config=config or {},
|
||||
dir=str(self.output_dir),
|
||||
reinit=True,
|
||||
)
|
||||
LOGGER.info("W&B initialized: %s", self._wandb_run.url)
|
||||
else:
|
||||
LOGGER.warning("wandb not installed, skipping W&B tracking")
|
||||
|
||||
# TensorBoard init.
|
||||
if use_tb:
|
||||
SummaryWriter = _try_import_tb()
|
||||
if SummaryWriter is not None:
|
||||
tb_dir = self.output_dir / "tb_logs"
|
||||
tb_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._tb_writer = SummaryWriter(log_dir=str(tb_dir))
|
||||
LOGGER.info("TensorBoard initialized: %s", tb_dir)
|
||||
else:
|
||||
LOGGER.warning("tensorboard not installed, skipping TB tracking")
|
||||
|
||||
@property
|
||||
def has_wandb(self) -> bool:
|
||||
return self._wandb_run is not None
|
||||
|
||||
@property
|
||||
def has_tb(self) -> bool:
|
||||
return self._tb_writer is not None
|
||||
|
||||
def log_train(self, epoch: int, metrics: dict[str, float], step: int | None = None) -> None:
|
||||
"""Log training metrics for an epoch."""
|
||||
if self._wandb_run is not None:
|
||||
self._wandb_run.log(
|
||||
{f"train/{k}": v for k, v in metrics.items()},
|
||||
step=step or epoch,
|
||||
)
|
||||
if self._tb_writer is not None:
|
||||
for k, v in metrics.items():
|
||||
self._tb_writer.add_scalar(f"train/{k}", v, global_step=step or epoch)
|
||||
|
||||
def log_val(self, epoch: int, metrics: dict[str, float], step: int | None = None) -> None:
|
||||
"""Log validation metrics."""
|
||||
if self._wandb_run is not None:
|
||||
self._wandb_run.log(
|
||||
{f"val/{k}": v for k, v in metrics.items()},
|
||||
step=step or epoch,
|
||||
)
|
||||
if self._tb_writer is not None:
|
||||
for k, v in metrics.items():
|
||||
self._tb_writer.add_scalar(f"val/{k}", v, global_step=step or epoch)
|
||||
|
||||
def log_gradients(self, epoch: int, grad_norms: dict[str, float], step: int | None = None) -> None:
|
||||
"""Log gradient norms per parameter group."""
|
||||
if self._wandb_run is not None:
|
||||
self._wandb_run.log(
|
||||
{f"gradients/{k}": v for k, v in grad_norms.items()},
|
||||
step=step or epoch,
|
||||
)
|
||||
if self._tb_writer is not None:
|
||||
for k, v in grad_norms.items():
|
||||
self._tb_writer.add_scalar(f"gradients/{k}", v, global_step=step or epoch)
|
||||
|
||||
def log_scalar(self, tag: str, value: float, step: int) -> None:
|
||||
"""Log a single scalar."""
|
||||
if self._wandb_run is not None:
|
||||
self._wandb_run.log({tag: value}, step=step)
|
||||
if self._tb_writer is not None:
|
||||
self._tb_writer.add_scalar(tag, value, global_step=step)
|
||||
|
||||
def log_image(self, tag: str, image: Any, step: int, caption: str | None = None) -> None:
|
||||
"""Log an image (numpy HWC or torch CHW).
|
||||
|
||||
Args:
|
||||
tag: Image tag/name.
|
||||
image: numpy array [H,W,C] or torch tensor [C,H,W].
|
||||
step: Global step.
|
||||
caption: Optional caption for W&B.
|
||||
"""
|
||||
if self._wandb_run is not None:
|
||||
wandb = _try_import_wandb()
|
||||
if isinstance(image, torch.Tensor):
|
||||
image_np = image.detach().cpu().permute(1, 2, 0).numpy()
|
||||
else:
|
||||
image_np = image
|
||||
self._wandb_run.log(
|
||||
{tag: wandb.Image(image_np, caption=caption)},
|
||||
step=step,
|
||||
)
|
||||
if self._tb_writer is not None:
|
||||
if isinstance(image, torch.Tensor):
|
||||
self._tb_writer.add_image(tag, image.detach().cpu(), global_step=step)
|
||||
else:
|
||||
self._tb_writer.add_image(tag, image, global_step=step, dataformats="HWC")
|
||||
|
||||
def log_histogram(self, tag: str, values: torch.Tensor, step: int) -> None:
|
||||
"""Log a histogram of values (weights, activations, etc.)."""
|
||||
if self._wandb_run is not None:
|
||||
wandb = _try_import_wandb()
|
||||
self._wandb_run.log(
|
||||
{tag: wandb.Histogram(values.detach().cpu().numpy())},
|
||||
step=step,
|
||||
)
|
||||
if self._tb_writer is not None:
|
||||
self._tb_writer.add_histogram(tag, values.detach().cpu(), global_step=step)
|
||||
|
||||
def log_model_graph(self, model: torch.nn.Module, input_example: Any = None) -> None:
|
||||
"""Log model graph to TensorBoard (if available)."""
|
||||
if self._tb_writer is not None and input_example is not None:
|
||||
try:
|
||||
self._tb_writer.add_graph(model, input_example)
|
||||
except Exception as e:
|
||||
LOGGER.warning("Failed to log model graph: %s", e)
|
||||
|
||||
def watch_model(self, model: torch.nn.Module, log_freq: int = 100) -> None:
|
||||
"""Enable W&B gradient/weight watching."""
|
||||
if self._wandb_run is not None:
|
||||
wandb = _try_import_wandb()
|
||||
wandb.watch(model, log="all", log_freq=log_freq)
|
||||
|
||||
def log_summary(self, summary: dict[str, Any]) -> None:
|
||||
"""Log final summary metrics (best R@1, etc.)."""
|
||||
if self._wandb_run is not None:
|
||||
for k, v in summary.items():
|
||||
self._wandb_run.summary[k] = v
|
||||
|
||||
def close(self) -> None:
|
||||
"""Flush and close all backends."""
|
||||
if self._tb_writer is not None:
|
||||
self._tb_writer.flush()
|
||||
self._tb_writer.close()
|
||||
if self._wandb_run is not None:
|
||||
self._wandb_run.finish()
|
||||
1297
vendor_reference/caption_test/src/training/train_gtauav.py
Normal file
1297
vendor_reference/caption_test/src/training/train_gtauav.py
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user