From 70bd4bb756bbd84fedfe79356d6d855e0b75eb42 Mon Sep 17 00:00:00 2001 From: Raghav Date: Thu, 9 Jul 2026 10:24:11 -0700 Subject: [PATCH] [bugfix]: don't let a stale HF-passthrough marker override a moved encoder (#1339) _fastvideo_input_device is stamped at load time with the loader's target device, which is "cpu" when text_encoder_cpu_offload is on. encode_text then moves such an encoder to the compute device for the forward, after which the marker is stale. Honouring it fed cpu token ids to cuda weights: RuntimeError: Expected all tensors to be on the same device, but got index is on cpu, different from other tensors on cuda:0 Seen on FLUX.2 Klein, whose Qwen3 text encoder takes the HF-passthrough path. Once the encoder has been moved, its param device is authoritative; the marker only speaks when nothing moved and the module has no parameters of its own. --- fastvideo/pipelines/stages/text_encoding.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/fastvideo/pipelines/stages/text_encoding.py b/fastvideo/pipelines/stages/text_encoding.py index b2e668136..68fb5bc52 100644 --- a/fastvideo/pipelines/stages/text_encoding.py +++ b/fastvideo/pipelines/stages/text_encoding.py @@ -221,12 +221,19 @@ class TextEncodingStage(PipelineStage): encoder_device = torch.device(target_device) moved_for_forward = True - # An explicit `device=` wins. Otherwise HF-passthrough encoders pin - # their own placement via _fastvideo_input_device (they return before - # the FSDP wrapping path, so encoder_device cannot speak for them); - # every other encoder follows its (possibly just-moved) param device. - input_device = (torch.device(target_device) if device is not None else getattr( - text_encoder, "_fastvideo_input_device", encoder_device)) + # An explicit `device=` wins. Otherwise follow the encoder's real + # param device. Once it has been moved for the forward that is the + # target device, and an HF-passthrough encoder's + # _fastvideo_input_device (stamped at load, e.g. "cpu" under + # text_encoder_cpu_offload) is stale -- honouring it would feed cpu + # token ids to cuda weights. The marker only speaks when nothing + # moved and the module has no parameters to speak for it. + if device is not None: + input_device = torch.device(target_device) + elif moved_for_forward: + input_device = encoder_device + else: + input_device = getattr(text_encoder, "_fastvideo_input_device", encoder_device) tok_kwargs = dict(encoder_config.tokenizer_kwargs) if max_length is not None: