forked from Pikaliov/fuze_task
2c6a00a4ca
Multimodal fusion research on StripNet+GTA-UAV proxy: - 3 independent fusion tracks: condition-aware (A), token/bottleneck (B), role-aware (C) - Shared interfaces, protocol, dataset audit, baseline benchmarks - Canonical version-chain references to vault (SPEC, ANALYSIS, TRIAGE) - Personalized task plans and decision tables for each researcher - 3 generated DOCX task assignment files with milestones and DoD checklist - Full modality dropout diagnostics and missing-modality robustness requirements - Data contract, benchmark registry, experiment tracking infrastructure Operational documents: - docs/00_project/: MERIDIAN context, protocol, repository reuse guide, experiment specification - docs/01_tasks/: Master assignment + 3 individual researcher tracks + joint integration - docs/02_references/: Core literature, version-chain bases, code maps - docs/03_codebase_guides/: Existing code snapshots from vault - scripts/: gen_task_plans.js (DOCX generation), placeholder infrastructure - vendor_reference/: Snapshots of caption_test, depth_edges_annotate, existing SOFIA/SegModel code - reports/, results/, experiments/: Shared output structure for all 3 researchers 3 DOCX files generated from gen_task_plans.js (Times New Roman 14pt, GOST format): - План_заданий_Павленко_БВ.docx (Condition-Aware track, fusion API owner) - План_заданий_Близно_МВ.docx (Token/Bottleneck track, benchmark owner) - План_заданий_Мороз_ЕС.docx (Role-Aware track, data contract owner) Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
61 lines
2.5 KiB
Python
61 lines
2.5 KiB
Python
from __future__ import annotations
|
|
|
|
"""Contract tests shared by all fusion implementations."""
|
|
|
|
import torch
|
|
|
|
from fuse_proj.data.types import FusionViewOutput, ViewBatch, ViewName
|
|
from fuse_proj.models.fusion.base import FusionModelBase
|
|
|
|
|
|
class DummyFusion(FusionModelBase):
|
|
"""Minimal parameter-free implementation used only to test the base contract."""
|
|
|
|
def encode_view(self, batch: ViewBatch, view: ViewName) -> FusionViewOutput:
|
|
"""Convert RGB channel means into a deterministic normalized descriptor."""
|
|
del view
|
|
rgb_summary = batch["rgb"].mean(dim=(-2, -1)) # [B, 3]
|
|
repeats = (self.descriptor_dim + rgb_summary.shape[1] - 1) // rgb_summary.shape[1]
|
|
raw = rgb_summary.repeat(1, repeats)[:, : self.descriptor_dim] # [B, D]
|
|
descriptor = self.normalize_descriptor(raw)
|
|
contributions = torch.zeros(raw.shape[0], 3, dtype=raw.dtype, device=raw.device)
|
|
return {
|
|
"descriptor": descriptor,
|
|
"rgb_descriptor": descriptor,
|
|
"modality_contributions": contributions,
|
|
"diagnostics": {},
|
|
}
|
|
|
|
|
|
def _make_batch(batch_size: int) -> ViewBatch:
|
|
"""Create a valid synthetic multimodal batch."""
|
|
return {
|
|
"rgb": torch.rand(batch_size, 3, 256, 256),
|
|
"geometry": torch.rand(batch_size, 1, 256, 256),
|
|
"segmentation": torch.zeros(batch_size, 1, 256, 256, dtype=torch.uint8),
|
|
"geometry_valid": torch.ones(batch_size, 1, 256, 256, dtype=torch.bool),
|
|
"segmentation_valid": torch.ones(batch_size, 1, 256, 256, dtype=torch.bool),
|
|
"text_valid": torch.ones(batch_size, dtype=torch.bool),
|
|
"text": ["urban scene"] * batch_size,
|
|
"sample_id": [f"sample-{index}" for index in range(batch_size)],
|
|
}
|
|
|
|
|
|
def test_pair_output_shapes() -> None:
|
|
"""The base contract must return one normalized descriptor per view."""
|
|
model = DummyFusion(descriptor_dim=1024)
|
|
output = model(_make_batch(4), _make_batch(4))
|
|
|
|
for view in ("satellite", "uav"):
|
|
descriptor = output[view]["descriptor"]
|
|
assert descriptor.shape == (4, 1024)
|
|
assert torch.allclose(descriptor.norm(dim=-1), torch.ones(4), atol=1e-5)
|
|
|
|
|
|
def test_batch_size_one() -> None:
|
|
"""Fusion implementations must not squeeze away the batch dimension."""
|
|
model = DummyFusion(descriptor_dim=1024)
|
|
output = model(_make_batch(1), _make_batch(1))
|
|
assert output["satellite"]["descriptor"].shape == (1, 1024)
|
|
assert output["uav"]["descriptor"].shape == (1, 1024)
|