Add gradient checkpointing for DINOv3 and DGTRS-CLIP (bs 8→24)

- DINOv3: checkpoint each of 24 transformer blocks (recompute on backward)
- DGTRS-CLIP: checkpoint each of 12 transformer blocks
- Enables batch_size=24 on RTX 4090 (was 8 without checkpointing)
- Peak VRAM: 20.3 GB at bs=24 (was OOM at bs=16 before)
- ~20-30% slower per step, but 3x more in-batch negatives (23 vs 7)
- Enabled by default (gradient_checkpointing=True in config)
- Update README with VRAM benchmarks and checkpointing docs

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pikaliov
2026-04-21 21:34:31 +03:00
parent da2d2ea90e
commit 6b7bcae198
5 changed files with 56 additions and 7 deletions

View File

@@ -169,12 +169,22 @@ class DINOv3ViT(nn.Module):
])
self.norm = nn.LayerNorm(dim)
self.embed_dim = dim
self.gradient_checkpointing = False
def set_gradient_checkpointing(self, enable: bool = True) -> None:
"""Enable/disable gradient checkpointing to trade compute for VRAM."""
self.gradient_checkpointing = enable
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""Forward pass. Returns CLS token embedding [B, dim]."""
x = self.embeddings(x)
for block in self.layer:
x = block(x)
if self.gradient_checkpointing and self.training:
x = torch.utils.checkpoint.checkpoint(
block, x, use_reentrant=False,
)
else:
x = block(x)
x = self.norm(x)
return x[:, 0] # CLS token

View File

@@ -77,8 +77,15 @@ class Transformer(nn.Module):
self.resblocks = nn.Sequential(
*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)]
)
self.gradient_checkpointing = False
def forward(self, x: torch.Tensor):
if self.gradient_checkpointing and self.training:
for block in self.resblocks:
x = torch.utils.checkpoint.checkpoint(
block, x, use_reentrant=False,
)
return x
return self.resblocks(x)