def _apply_cap(self, activations, vector, tau):
vector = vector.to(activations.device)
v = vector / (vector.norm() + 1e-8)
if self.positions == "all":
proj = torch.einsum('bld,d->bl', activations, v)
excess = (proj - tau).clamp(min=0.0)
return activations - torch.einsum('bl,d->bld', excess, v)
From the idea of the paper, I thought it should also include raising the lower part like below:
def _apply_cap_bilateral(self, activations, vector, tau_low, tau_high):
vector = vector.to(activations.device)
v = vector / (vector.norm() + 1e-8)
if self.positions == "all":
proj = torch.einsum('bld,d->bl', activations, v)
excess_high = (proj - tau_high).clamp(min=0.0)
excess_low = (tau_low - proj).clamp(min=0.0)
return activations - torch.einsum('bl,d->bld', excess_high, v) + torch.einsum('bl,d->bld', excess_low, v)
def _apply_cap(self, activations, vector, tau):
vector = vector.to(activations.device)
v = vector / (vector.norm() + 1e-8)
From the idea of the paper, I thought it should also include raising the lower part like below:
def _apply_cap_bilateral(self, activations, vector, tau_low, tau_high):
vector = vector.to(activations.device)
v = vector / (vector.norm() + 1e-8)