From 9f76a4fe0983d72a4005bb81d7d07da278c6a67c Mon Sep 17 00:00:00 2001 From: Martin Bukowski Date: Mon, 22 Jan 2024 14:38:52 -0600 Subject: [PATCH] normalize --- README.md | 2 +- components/normalize.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2986dd4..576e400 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,4 @@ DARE-TIES does a stochastic selection of the parameters to keep, and then only p * Take model B, and merge it in to model A using the mask to protect model A's largest parameters. ### Normalization -I am testing out a new normalization method, which is to normalize the norm of the parameters of one model to another. This is done by taking the ratio of the norms, and then scaling the parameters of the first model by that ratio. This is done in the `Normalize Model` node. There are a few options, of most interest is the 'q_norm' option, which only scales Q and K relative to each other. +I am testing out a new normalization method, which is to normalize the norm of the parameters of one model to another. This is done by taking the ratio of the norms, and then scaling the parameters of the first model by that ratio. This is done in the `Normalize Model` node. There are a few options, of most interest is the 'attn_only' option, which only scales Q and K relative to each other, and just that. diff --git a/components/normalize.py b/components/normalize.py index 7669683..4834691 100644 --- a/components/normalize.py +++ b/components/normalize.py @@ -36,7 +36,7 @@ class NormalizeUnet: "required": { "model_a": ("MODEL",), "model_b": ("MODEL",), - "method": (["q_norm", "all", "none", "attn_only"], ), + "method": (["q_norm", "all", "none", "attn_only"], {"default": "attn_only"}), } }