simplified

2024-06-07 07:05:15 +02:00
parent c85b2cd9a3
commit 9530e2eb39
1 changed files with 2 additions and 1 deletions
@@ -39,10 +39,11 @@ class temperature_patcher():
        
        out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False, scale=scale)
        if should_scale(self.model_name, self.layer_name,q.size(-2)):
+            ldim = SD_layer_dims[self.model_name][self.layer_name]
            if self.eval_string != "":
                out = eval(self.eval_string)
            else:
-                out *= math.log(q.size(-2)) / math.log(SD_layer_dims[self.model_name][self.layer_name])
+                out *= math.log(q.size(-2), ldim)

        out = (
            out.transpose(1, 2).reshape(b, -1, heads * dim_head)