diff --git a/src/training/train_gtauav.py b/src/training/train_gtauav.py index 4e1b16e..7127022 100644 --- a/src/training/train_gtauav.py +++ b/src/training/train_gtauav.py @@ -839,15 +839,20 @@ def train(cfg: TrainConfigGTAUAV) -> None: ) loss_dict = loss_fn(**loss_kwargs) - # Enqueue current gallery embeddings (detached). - if neg_bank is not None: - neg_bank.enqueue(embeddings["gallery"].detach()) - # Scale loss by accumulation steps so gradients average correctly. raw_loss = float(loss_dict["total"].item()) # save before backward total_loss = loss_dict["total"] / accum scaler.scale(total_loss).backward() + # Enqueue current gallery AFTER backward. The queue buffer is aliased + # into the autograd graph through `queue_neg` (a view returned by + # `NegativeMemoryBank.get_queue`), so modifying it before backward + # triggers "variable needed for gradient computation has been modified + # by an inplace operation". Enqueueing here is semantically identical + # — the next step's queue state is the same either way. + if neg_bank is not None: + neg_bank.enqueue(embeddings["gallery"].detach()) + # Optimizer step only after accumulating `accum` micro-batches. is_accum_step = (n_batches + 1) % accum == 0 or (n_batches + 1) == len(train_loader) if is_accum_step: