fix: lora_B init zeros (#781)

Co-authored-by: zbchu2 <zbchu2@iflytek.com> Co-authored-by: Wenxuan Tan <wenxuan.tan@wisc.edu>
2025-09-05 22:56:52 -07:00
parent d3ceb67e66
commit 85eb7265db
1 changed files with 1 additions and 1 deletions
@@ -63,7 +63,7 @@ class BaseLayerWithLoRA(nn.Module):
                            device=self.base_layer.weight.device,
                            dtype=self.base_layer.weight.dtype))
            torch.nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
-            torch.nn.init.kaiming_uniform_(self.lora_B, a=math.sqrt(5))
+            torch.nn.init.zeros_(self.lora_B)
        else:
            self.lora_A = None
            self.lora_B = None