Revert using torch RMSNorm as there are quality concerns

2025-09-21 18:19:27 +03:00
parent fc3a684b0b
commit 718946ef13
1 changed files with 1 additions and 1 deletions
@@ -407,7 +407,7 @@ class WanSelfAttention(nn.Module):
            self.v = nn.Linear(in_features, out_features)
        self.o = nn.Linear(in_features, out_features)

-        if hasattr(torch.nn, 'RMSNorm'):
+        if False: # disable for now, seems to degrade quality
            self.norm_q = WanFusedRMSNorm(out_features, eps=eps) if qk_norm else nn.Identity()
            self.norm_k = WanFusedRMSNorm(out_features, eps=eps) if qk_norm else nn.Identity()
        else: