From 9ab9fe0bfcb70d483109d91dd649bffbe7d4946f Mon Sep 17 00:00:00 2001 From: Extraltodeus Date: Thu, 6 Jun 2024 16:54:31 +0200 Subject: [PATCH] Update nodes.py --- nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index 982e82f..6f36959 100644 --- a/nodes.py +++ b/nodes.py @@ -36,7 +36,7 @@ class temperature_patcher(): out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False, scale=scale) if should_scale(self.model_name, self.layer_name,q.size(-2)): - out *= math.log(q.size(-2) ** 0.5) / math.log(SD_layer_dims[self.model_name][self.layer_name] ** 0.5) + out *= math.log(q.size(-2)) / math.log(SD_layer_dims[self.model_name][self.layer_name]) out = ( out.transpose(1, 2).reshape(b, -1, heads * dim_head)