From ea4b2d31956c9f002f37f490e1b5257dcbdeb548 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Sun, 31 May 2026 02:01:05 +0000 Subject: [PATCH] some fix --- fastvideo/configs/pipelines/glm_image.py | 2 +- fastvideo/models/dits/glm_image.py | 2 +- .../pipelines/basic/glm_image/stages/before_denoising.py | 9 +++++---- .../basic/glm_image/stages/condition_encoding.py | 2 ++ fastvideo/pipelines/basic/glm_image/stages/denoising.py | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/fastvideo/configs/pipelines/glm_image.py b/fastvideo/configs/pipelines/glm_image.py index 7f759d64b..7b2f46279 100644 --- a/fastvideo/configs/pipelines/glm_image.py +++ b/fastvideo/configs/pipelines/glm_image.py @@ -18,8 +18,8 @@ def glm_image_t5_postprocess(outputs: BaseEncoderOutput) -> torch.Tensor: assert torch.isnan(hidden_state).sum() == 0, "T5 hidden states contain NaN" - prompt_embeds = [u[:v] for u, v in zip(hidden_state, seq_lens, strict=True)] max_len = 512 + prompt_embeds = [u[:min(v, max_len)] for u, v in zip(hidden_state, seq_lens, strict=True)] prompt_embeds_tensor: torch.Tensor = torch.stack( [torch.cat([u, u.new_zeros(max_len - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0) diff --git a/fastvideo/models/dits/glm_image.py b/fastvideo/models/dits/glm_image.py index ae5255cb5..3dbc2642e 100644 --- a/fastvideo/models/dits/glm_image.py +++ b/fastvideo/models/dits/glm_image.py @@ -737,7 +737,7 @@ class GlmImageTransformer2DModel(BaseDiT): encoder_hidden_states = self.glyph_projector(encoder_hidden_states) prior_embedding = self.prior_token_embedding(prior_token_id) # Zero dropped priors by multiply: boolean indexing + .any() syncs each step. - keep = (~prior_token_drop).to(prior_embedding.dtype) + keep = (~prior_token_drop).to(device=prior_embedding.device, dtype=prior_embedding.dtype) while keep.dim() < prior_embedding.dim(): keep = keep.unsqueeze(-1) prior_embedding = prior_embedding * keep diff --git a/fastvideo/pipelines/basic/glm_image/stages/before_denoising.py b/fastvideo/pipelines/basic/glm_image/stages/before_denoising.py index aeb24e5b2..cce0b9f7d 100644 --- a/fastvideo/pipelines/basic/glm_image/stages/before_denoising.py +++ b/fastvideo/pipelines/basic/glm_image/stages/before_denoising.py @@ -161,13 +161,14 @@ class GlmImageBeforeDenoisingStage(PipelineStage): outputs = self.vision_language_encoder.generate(**inputs, max_new_tokens=max_new, do_sample=True) gen_tokens = outputs[0][inputs.input_ids.shape[-1]:] - if len(gen_tokens) >= large_start + large_count: + if gen_tokens.shape[0] >= large_start + large_count: large_tokens = gen_tokens[large_start:large_start + large_count] else: - available = gen_tokens[large_start:] if len(gen_tokens) > large_start else gen_tokens + available = gen_tokens[large_start:] large_tokens = torch.zeros(large_count, dtype=gen_tokens.dtype, device=gen_tokens.device) - large_tokens[:min(len(available), large_count)] = available[:large_count] - logger.warning("AR generated %d tokens, expected %d. Padding with zeros.", len(gen_tokens), + if available.shape[0] > 0: + large_tokens[:min(available.shape[0], large_count)] = available[:large_count] + logger.warning("AR generated %d tokens, expected %d. Padding with zeros.", gen_tokens.shape[0], large_start + large_count) batch.prior_token_id = _upsample_d32_to_d16(large_tokens, up_h, up_w) batch.prior_token_drop = torch.zeros(batch.prior_token_id.shape, dtype=torch.bool, device=device) diff --git a/fastvideo/pipelines/basic/glm_image/stages/condition_encoding.py b/fastvideo/pipelines/basic/glm_image/stages/condition_encoding.py index b5f8b5ecf..8fe8ef347 100644 --- a/fastvideo/pipelines/basic/glm_image/stages/condition_encoding.py +++ b/fastvideo/pipelines/basic/glm_image/stages/condition_encoding.py @@ -42,6 +42,8 @@ class GlmImageConditionEncodingStage(PipelineStage): cond_img = self.image_processor.preprocess(batch.pil_image, cond_h, cond_w).to(device=device, dtype=torch.float32) latent = self.vae.encode(cond_img).latent_dist.mode() + # NOTE: at runtime self.vae.config is a diffusers FrozenDict with flat + # latents_mean/latents_std fields. Access the flat fields directly. cfg = self.vae.config mean = torch.tensor(cfg.latents_mean, device=device, dtype=torch.float32).view(1, -1, 1, 1) std = torch.tensor(cfg.latents_std, device=device, dtype=torch.float32).view(1, -1, 1, 1) diff --git a/fastvideo/pipelines/basic/glm_image/stages/denoising.py b/fastvideo/pipelines/basic/glm_image/stages/denoising.py index b6a6ff29e..db2e98d55 100644 --- a/fastvideo/pipelines/basic/glm_image/stages/denoising.py +++ b/fastvideo/pipelines/basic/glm_image/stages/denoising.py @@ -133,7 +133,7 @@ class GlmImageDenoisingStage(DenoisingStage): prior_token_id = batch.prior_token_id[:1] drop_keep = torch.zeros((1, ), dtype=torch.bool, device=device) drop_all = torch.ones((1, ), dtype=torch.bool, device=device) - cache_len = kv_caches[0].k_cache.shape[1] + cache_len = kv_caches[0].k_cache.shape[1] if kv_caches[0].k_cache is not None else 0 def _mask(row: int, with_cache: bool): if not sdpa: