From 4f18ee7a2d9dbe7334e2d7615f30aa94495b0c2d Mon Sep 17 00:00:00 2001 From: JTriggerFish Date: Thu, 20 Feb 2025 00:04:11 +0000 Subject: [PATCH] Mean fix --- core/guidance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/guidance.py b/core/guidance.py index 203fe7b..a81d46c 100644 --- a/core/guidance.py +++ b/core/guidance.py @@ -616,6 +616,7 @@ def value_rescale_attention_wrapper( Randomly permute the values before computing attention. """ orig_dtype = v.dtype + mean = v.mean() global_scaling = scaling_factor * torch.exp( torch.randn(1, device=v.device) * global_random_scaling_std - 0.5 * global_random_scaling_std**2 @@ -624,7 +625,7 @@ def value_rescale_attention_wrapper( torch.randn(v.shape, device=v.device) * local_random_scaling_std - 0.5 * local_random_scaling_std**2 ).clamp(0.01, 100.0) - new_v = global_scaling * local_scaling * v + new_v = global_scaling * local_scaling * (v - mean) + mean new_v = new_v.to(orig_dtype) return optimized_attention(q, k, new_v, heads=heads)