Add gradient checkpointing for DINOv3 and DGTRS-CLIP (bs 8→24)
- DINOv3: checkpoint each of 24 transformer blocks (recompute on backward) - DGTRS-CLIP: checkpoint each of 12 transformer blocks - Enables batch_size=24 on RTX 4090 (was 8 without checkpointing) - Peak VRAM: 20.3 GB at bs=24 (was OOM at bs=16 before) - ~20-30% slower per step, but 3x more in-batch negatives (23 vs 7) - Enabled by default (gradient_checkpointing=True in config) - Update README with VRAM benchmarks and checkpointing docs Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -169,12 +169,22 @@ class DINOv3ViT(nn.Module):
|
||||
])
|
||||
self.norm = nn.LayerNorm(dim)
|
||||
self.embed_dim = dim
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def set_gradient_checkpointing(self, enable: bool = True) -> None:
|
||||
"""Enable/disable gradient checkpointing to trade compute for VRAM."""
|
||||
self.gradient_checkpointing = enable
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Forward pass. Returns CLS token embedding [B, dim]."""
|
||||
x = self.embeddings(x)
|
||||
for block in self.layer:
|
||||
x = block(x)
|
||||
if self.gradient_checkpointing and self.training:
|
||||
x = torch.utils.checkpoint.checkpoint(
|
||||
block, x, use_reentrant=False,
|
||||
)
|
||||
else:
|
||||
x = block(x)
|
||||
x = self.norm(x)
|
||||
return x[:, 0] # CLS token
|
||||
|
||||
|
||||
@@ -77,8 +77,15 @@ class Transformer(nn.Module):
|
||||
self.resblocks = nn.Sequential(
|
||||
*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)]
|
||||
)
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
if self.gradient_checkpointing and self.training:
|
||||
for block in self.resblocks:
|
||||
x = torch.utils.checkpoint.checkpoint(
|
||||
block, x, use_reentrant=False,
|
||||
)
|
||||
return x
|
||||
return self.resblocks(x)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user