From df2cfbeefca8e295b4b3fe700e3b4156574670a2 Mon Sep 17 00:00:00 2001 From: Mihir Jagtap Date: Sun, 8 Feb 2026 21:23:13 -0800 Subject: [PATCH] Update rotary_embedding.py # For rotate_half style RoPE (used by HunyuanVideo, GameCraft), # we need to expand cos/sin to full head_dim using repeat_interleave. # The rotate_half operation works on consecutive PAIRS: (x0,x1), (x2,x3)... # so cos/sin must be interleaved: [c0,c0,c1,c1,...] to match the pairing. # Using torch.cat would produce [c0,c1,...,c0,c1,...] which is WRONG. --- fastvideo/layers/rotary_embedding.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fastvideo/layers/rotary_embedding.py b/fastvideo/layers/rotary_embedding.py index 1a5c60d61..0db05cb1c 100644 --- a/fastvideo/layers/rotary_embedding.py +++ b/fastvideo/layers/rotary_embedding.py @@ -332,13 +332,13 @@ def get_1d_rotary_pos_embed( freqs_sin = freqs.sin() # [S, D/2] if use_real: - # For rotate_half style RoPE (used by HunyuanVideo, GameCraft), - # we need to repeat cos/sin to full head_dim - # The rotate_half operation expects: x * cos + rotate_half(x) * sin - # where rotate_half splits x into [x1, x2] and returns [-x2, x1] - # So we need cos and sin to have shape [S, D] to broadcast correctly - freqs_cos = torch.cat([freqs_cos, freqs_cos], dim=-1) # [S, D] - freqs_sin = torch.cat([freqs_sin, freqs_sin], dim=-1) # [S, D] + # For rotate_half style RoPE (used by HunyuanVideo, GameCraft), + # we need to expand cos/sin to full head_dim using repeat_interleave. + # The rotate_half operation works on consecutive PAIRS: (x0,x1), (x2,x3)... + # so cos/sin must be interleaved: [c0,c0,c1,c1,...] to match the pairing. + # Using torch.cat would produce [c0,c1,...,c0,c1,...] which is WRONG. + freqs_cos = freqs_cos.repeat_interleave(2, dim=-1) # [S, D] + freqs_sin = freqs_sin.repeat_interleave(2, dim=-1) # [S, D] return freqs_cos, freqs_sin