diff --git a/clipperpweight.py b/clipperpweight.py index ab07e47..47c83d0 100644 --- a/clipperpweight.py +++ b/clipperpweight.py @@ -15,9 +15,7 @@ class CLIPTextEncodePerpWeight: def encode(self, clip, text): empty_tokens = clip.tokenize("") - sdxl_flag = False - if isinstance(empty_tokens, dict): - sdxl_flag = True + sdxl_flag = isinstance(empty_tokens, dict) if sdxl_flag: empty_cond, empty_cond_pooled = clip.encode_from_tokens(empty_tokens, return_pooled=True) @@ -30,10 +28,10 @@ class CLIPTextEncodePerpWeight: cond = torch.clone(unweighted_cond) for i in range(unweighted_cond.shape[0]): for j in range(unweighted_cond.shape[1]): - weight_l = tokens["l"][i][j][1] + weight_l = tokens["l"][(j//77)][(j%77)][1] if weight_l != 1.0: token_vector_l = unweighted_cond[i][j][:768] - zero_vector_l = empty_cond[0][j][:768] + zero_vector_l = empty_cond[0][(j%77)][:768] perp_l = ((torch.mul(zero_vector_l, token_vector_l).sum())/(torch.norm(token_vector_l)**2)) * token_vector_l if weight_l > 1.0: cond[i][j][:768] = token_vector_l + (weight_l * perp_l) @@ -42,12 +40,12 @@ class CLIPTextEncodePerpWeight: elif weight_l < 0.0: cond[i][j][:768] = token_vector_l + (weight_l * perp_l) elif weight_l == 0.0: - cond[i][j][:768] = empty_cond[0][j][:768] + cond[i][j][:768] = empty_cond[0][(j%77)][:768] - weight_g = tokens["g"][i][j][1] + weight_g = tokens["g"][(j//77)][(j%77)][1] if weight_g != 1.0: token_vector_g = unweighted_cond[i][j][768:] - zero_vector_g = empty_cond[0][j][768:] + zero_vector_g = empty_cond[0][(j%77)][768:] perp_g = ((torch.mul(zero_vector_g, token_vector_g).sum())/(torch.norm(token_vector_g)**2)) * token_vector_g if weight_g > 1.0: cond[i][j][768:] = token_vector_g + (weight_g * perp_g) @@ -56,7 +54,7 @@ class CLIPTextEncodePerpWeight: elif (weight_g < 0.0): cond[i][j][768:] = token_vector_g + (weight_g * perp_g) elif weight_g == 0.0: - cond[i][j][768:] = empty_cond[0][j][768:] + cond[i][j][768:] = empty_cond[0][(j%77)][768:] else: empty_cond, empty_cond_pooled = clip.encode_from_tokens(empty_tokens, return_pooled=True) tokens = clip.tokenize(text) @@ -66,10 +64,10 @@ class CLIPTextEncodePerpWeight: cond = torch.clone(unweighted_cond) for i in range(unweighted_cond.shape[0]): for j in range(unweighted_cond.shape[1]): - weight = tokens[i][j][1] + weight = tokens[(j//77)][(j%77)][1] if weight != 1.0: token_vector = unweighted_cond[i][j] - zero_vector = empty_cond[0][j] + zero_vector = empty_cond[0][(j%77)] perp = ((torch.mul(zero_vector, token_vector).sum())/(torch.norm(token_vector)**2)) * token_vector if weight > 1.0: cond[i][j] = token_vector + (weight * perp) @@ -78,7 +76,7 @@ class CLIPTextEncodePerpWeight: elif (weight < 0.0): cond[i][j] = token_vector + (weight * perp) elif weight == 0.0: - cond[i][j] = empty_cond[0][j] + cond[i][j] = empty_cond[0][(j%77)] return ([[cond, {"pooled_output": unweighted_pooled}]], )