diff --git a/examples/inference/basic/basic_kandinsky5_i2v.py b/examples/inference/basic/basic_kandinsky5_i2v.py index b3b6efebb..665dd19d5 100644 --- a/examples/inference/basic/basic_kandinsky5_i2v.py +++ b/examples/inference/basic/basic_kandinsky5_i2v.py @@ -1,9 +1,10 @@ from fastvideo import VideoGenerator -OUTPUT_PATH = "video_samples_kandinsky5_2B_t2v" +OUTPUT_PATH = "video_samples_kandinsky5_2B_i2v" IMAGE_PATH = "assets/girl.png" + def main(): generator = VideoGenerator.from_pretrained( "kandinskylab/Kandinsky-5.0-I2V-Lite-sft-5s-Diffusers", @@ -22,8 +23,16 @@ def main(): "brings it to her lips to take a sip. After drinking, she places the cup back on " "the table and looks out the window, enjoying the peaceful atmosphere." ) - _ = generator.generate_video(prompt, output_path=OUTPUT_PATH, save_video=True,height=512, width=768, num_frames=121) + _ = generator.generate_video( + prompt, + image_path=IMAGE_PATH, + output_path=OUTPUT_PATH, + save_video=True, + height=512, + width=768, + num_frames=121, + ) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/fastvideo/configs/pipelines/__init__.py b/fastvideo/configs/pipelines/__init__.py index 55b396af3..3983479de 100644 --- a/fastvideo/configs/pipelines/__init__.py +++ b/fastvideo/configs/pipelines/__init__.py @@ -6,7 +6,7 @@ from fastvideo.configs.pipelines.hunyuan import FastHunyuanConfig, HunyuanConfig from fastvideo.configs.pipelines.hunyuan15 import Hunyuan15T2V480PConfig, Hunyuan15T2V720PConfig from fastvideo.configs.pipelines.hunyuangamecraft import HunyuanGameCraftPipelineConfig from fastvideo.configs.pipelines.hyworld import HYWorldConfig -from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5T2VConfig +from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5I2VConfig, Kandinsky5T2VConfig from fastvideo.configs.pipelines.matrixgame2 import MatrixGame2I2V480PConfig from fastvideo.configs.pipelines.matrixgame3 import MatrixGame3I2V720PConfig from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig @@ -19,5 +19,5 @@ __all__ = [ "Hunyuan15T2V720PConfig", "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig", "SelfForcingWanT2V480PConfig", "LucyEditDevConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig", "DreamXWorld5BCamPipelineConfig", "DreamXWorld5BARPipelineConfig", "HYWorldConfig", "Kandinsky5T2VConfig", - "MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name" + "Kandinsky5I2VConfig", "MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name" ] diff --git a/fastvideo/configs/pipelines/kandinsky5.py b/fastvideo/configs/pipelines/kandinsky5.py index 1b3b1e308..6981c2b70 100644 --- a/fastvideo/configs/pipelines/kandinsky5.py +++ b/fastvideo/configs/pipelines/kandinsky5.py @@ -101,3 +101,13 @@ class Kandinsky5T2VConfig(PipelineConfig): "add_special_tokens": True, "return_tensors": "pt", }) + + +@dataclass +class Kandinsky5I2VConfig(Kandinsky5T2VConfig): + """Kandinsky-5.0 image-to-video pipeline configuration.""" + + def __post_init__(self) -> None: + super().__post_init__() + # I2V needs the VAE encoder to encode the conditioning image. + self.vae_config.load_encoder = True diff --git a/fastvideo/pipelines/basic/kandinsky5/kandinsky5_i2v_pipeline.py b/fastvideo/pipelines/basic/kandinsky5/kandinsky5_i2v_pipeline.py index 09605fa9e..b34426c58 100644 --- a/fastvideo/pipelines/basic/kandinsky5/kandinsky5_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/kandinsky5/kandinsky5_i2v_pipeline.py @@ -8,7 +8,9 @@ from fastvideo.pipelines.stages.input_validation import InputValidationStage from fastvideo.pipelines.stages.kandinsky5 import ( Kandinsky5DecodingStage, Kandinsky5DenoisingStage, + Kandinsky5ImageEncodingStage, Kandinsky5LatentPreparationStage, + Kandinsky5NormalizationStage, ) from fastvideo.pipelines.stages.text_encoding import TextEncodingStage from fastvideo.pipelines.stages.timestep_preparation import TimestepPreparationStage @@ -28,8 +30,6 @@ class Kandinsky5I2VPipeline(ComposedPipelineBase): ] def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: - raise NotImplementedError("Class to be implemented yet") - self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) self.add_stage( @@ -45,10 +45,16 @@ class Kandinsky5I2VPipeline(ComposedPipelineBase): ], ), ) - - self.add_stage(stage_name="image_encoding_stage", - stage=Kandinsky5ImageEncodingStage(vae=self.get_module("vae"))) + self.add_stage( + stage_name="image_encoding_stage", + stage=Kandinsky5ImageEncodingStage(vae=self.get_module("vae")), + ) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")), + ) self.add_stage( stage_name="latent_preparation_stage", @@ -57,11 +63,6 @@ class Kandinsky5I2VPipeline(ComposedPipelineBase): transformer=self.get_module("transformer"), ), ) - - self.add_stage( - stage_name="decoding_stage", - stage=Kandinsky5DecodingStage(vae=self.get_module("vae"), pipeline=self), - ) self.add_stage( stage_name="denoising_stage", @@ -71,10 +72,15 @@ class Kandinsky5I2VPipeline(ComposedPipelineBase): ), ) + self.add_stage( + stage_name="normalization_stage", + stage=Kandinsky5NormalizationStage(), + ) + self.add_stage( stage_name="decoding_stage", stage=Kandinsky5DecodingStage(vae=self.get_module("vae"), pipeline=self), ) -EntryClass = Kandinsky5T2VPipeline +EntryClass = Kandinsky5I2VPipeline diff --git a/fastvideo/pipelines/basic/kandinsky5/presets.py b/fastvideo/pipelines/basic/kandinsky5/presets.py index 4864d35ae..f75db1760 100644 --- a/fastvideo/pipelines/basic/kandinsky5/presets.py +++ b/fastvideo/pipelines/basic/kandinsky5/presets.py @@ -34,4 +34,22 @@ KANDINSKY5_T2V_LITE_5S = InferencePreset( }, ) -ALL_PRESETS = (KANDINSKY5_T2V_LITE_5S, ) +KANDINSKY5_I2V_LITE_5S = InferencePreset( + name="kandinsky5_i2v_lite_5s", + version=1, + model_family="kandinsky5", + description="Kandinsky-5.0 Lite I2V 5s", + workload_type="i2v", + stage_schemas=(_DENOISE_STAGE, ), + defaults={ + "height": 512, + "width": 768, + "num_frames": 121, + "fps": 24, + "guidance_scale": 5.0, + "num_inference_steps": 50, + "negative_prompt": _NEGATIVE_PROMPT, + }, +) + +ALL_PRESETS = (KANDINSKY5_T2V_LITE_5S, KANDINSKY5_I2V_LITE_5S) diff --git a/fastvideo/pipelines/stages/kandinsky5.py b/fastvideo/pipelines/stages/kandinsky5.py index 0c3b19280..40a06fdcd 100644 --- a/fastvideo/pipelines/stages/kandinsky5.py +++ b/fastvideo/pipelines/stages/kandinsky5.py @@ -4,6 +4,7 @@ from __future__ import annotations import contextlib +import PIL import torch from diffusers.utils.torch_utils import randn_tensor from tqdm.auto import tqdm @@ -12,6 +13,7 @@ from fastvideo.distributed import get_local_torch_device from fastvideo.fastvideo_args import FastVideoArgs from fastvideo.models.loader.component_loader import TransformerLoader, VAELoader from fastvideo.models.vaes.common import ParallelTiledVAE +from fastvideo.models.vision_utils import normalize, numpy_to_pt, pil_to_numpy, resize from fastvideo.pipelines.pipeline_batch_info import ForwardBatch from fastvideo.pipelines.stages.base import PipelineStage from fastvideo.pipelines.stages.decoding import DecodingStage @@ -82,23 +84,24 @@ class Kandinsky5LatentPreparationStage(PipelineStage): else: latents = batch.latents.to(device=device, dtype=dtype) - if getattr(self.transformer, "visual_cond", False) and latents.shape[-1] == num_channels: - visual_cond = torch.zeros_like(latents) - visual_cond_mask = torch.zeros( + visual_cond = getattr(self.transformer, "visual_cond", False) + if visual_cond and latents.shape[-1] == num_channels: + cond = torch.zeros_like(latents) + cond_mask = torch.zeros( (*latents.shape[:-1], 1), device=latents.device, dtype=latents.dtype, ) - latents = torch.cat([latents, visual_cond, visual_cond_mask], dim=-1) - - if batch.image_latent is not None: - try: - raise NotImplementedError("Class to be implemented for I2V") - except NotImplementedError as e: - print(e) - else: - logger.warning("No image_latent found in batch, proceeding without conditioning") + latents = torch.cat([latents, cond, cond_mask], dim=-1) + # I2V: place the encoded image in the first frame and, when the + # transformer expects visual conditioning, in the cond/mask channels. + if batch.image_latent is not None: + image_latent = batch.image_latent.to(device=device, dtype=dtype) + latents[:, 0:1, :, :, :num_channels] = image_latent + if visual_cond: + latents[:, 0:1, :, :, num_channels:2 * num_channels] = image_latent + latents[:, 0:1, :, :, 2 * num_channels:] = 1.0 batch.latents = latents batch.raw_latent_shape = ( @@ -273,9 +276,12 @@ class Kandinsky5DenoisingStage(PipelineStage): torch.arange(width // spatial_ratio // 2, device=device), ] scale_factor = self._scale_factor(height, width) - + sparse_params = self.get_sparse_params(latents, device) + # I2V keeps the first (conditioning) frame fixed during denoising. + cond_frames = 1 if getattr(self.transformer, "visual_cond", False) else 0 + with tqdm(total=batch.num_inference_steps, desc="Kandinsky5 Denoising") as progress_bar: for i, timestep in enumerate(batch.timesteps): if hasattr(self, "interrupt") and self.interrupt: @@ -312,10 +318,10 @@ class Kandinsky5DenoisingStage(PipelineStage): pred_velocity = uncond_pred_velocity + batch.guidance_scale * (pred_velocity - uncond_pred_velocity) - latents[:, :, :, :, :num_channels] = self.scheduler.step( - pred_velocity, + latents[:, cond_frames:, :, :, :num_channels] = self.scheduler.step( + pred_velocity[:, cond_frames:], timestep, - latents[:, :, :, :, :num_channels], + latents[:, cond_frames:, :, :, :num_channels], return_dict=False, )[0] @@ -358,9 +364,83 @@ class Kandinsky5DecodingStage(DecodingStage): class Kandinsky5ImageEncodingStage(EncodingStage): + """Encode the conditioning image into a VAE latent for I2V.""" def __init__(self, vae: ParallelTiledVAE, pipeline=None) -> None: - super().__init__(vae=vae, pipeline=pipeline) + super().__init__(vae=vae) + + @staticmethod + def _preprocess(image, height: int, width: int) -> torch.Tensor: + if isinstance(image, PIL.Image.Image): + image = resize(image, height, width) + image = numpy_to_pt(pil_to_numpy(image)) + if image.min() >= 0: + image = normalize(image) # [0, 1] -> [-1, 1] + return image + + @torch.no_grad() + def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: + if batch.pil_image is None: + raise ValueError("Kandinsky5 I2V requires an input image.") + + device = get_local_torch_device() + self.vae = self.vae.to(device) + vae_dtype = PRECISION_TO_TYPE[fastvideo_args.pipeline_config.vae_precision] + vae_autocast_enabled = vae_dtype != torch.float32 and not fastvideo_args.disable_autocast + + # [B, C, H, W] -> [B, C, 1, H, W] + image = self._preprocess(batch.pil_image, int(batch.height), int(batch.width)) + image = image.to(device=device, dtype=torch.float32).unsqueeze(2) + + with torch.autocast(device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled): + if not vae_autocast_enabled: + image = image.to(vae_dtype) + image_latent = self.vae.encode(image).mode() + + image_latent = image_latent * self.vae.scaling_factor + # [B, C, 1, H, W] -> [B, 1, H, W, C] to match channels-last latents + batch.image_latent = image_latent.permute(0, 2, 3, 4, 1).contiguous() + + if fastvideo_args.vae_cpu_offload: + self.vae.to("cpu") + return batch + + def verify_input(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("pil_image", batch.pil_image, V.not_none) + result.add_check("height", batch.height, V.positive_int) + result.add_check("width", batch.width, V.positive_int) + return result + + def verify_output(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("image_latent", batch.image_latent, [V.is_tensor, V.with_dims(5)]) + return result + + +class Kandinsky5NormalizationStage(PipelineStage): + """Normalize the first latent frames to reduce I2V conditioning artifacts.""" + + COND_FRAMES = 4 + REFERENCE_FRAMES = 5 + + @staticmethod + def _adaptive_mean_std(source: torch.Tensor, reference: torch.Tensor) -> torch.Tensor: + source_mean = source.mean(dim=(1, 2, 3, 4), keepdim=True) + source_std = source.std(dim=(1, 2, 3, 4), keepdim=True) + # Magic constants limit how far the first frames may drift. + ref_mean = torch.clamp(reference.mean(), source_mean - 0.05, source_mean + 0.1) + ref_std = torch.clamp(reference.std(), source_std - 0.1, source_std + 0.25) + normalized = (source - source_mean) / source_std + return normalized * ref_std + ref_mean def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: - raise NotImplementedError("Class to be implemented for I2V") + latents = batch.latents + n = self.COND_FRAMES + if latents is None or latents.shape[1] <= n: + return batch + + reference = latents[:, n:n + min(self.REFERENCE_FRAMES, latents.shape[1] - 1)] + latents[:, :n] = self._adaptive_mean_std(latents[:, :n].clone(), reference) + batch.latents = latents + return batch diff --git a/fastvideo/registry.py b/fastvideo/registry.py index deb610c93..7fe0e2bd7 100644 --- a/fastvideo/registry.py +++ b/fastvideo/registry.py @@ -28,7 +28,7 @@ from fastvideo.configs.pipelines.hunyuan15 import (Hunyuan15T2V480PConfig, Hunyu Hunyuan15T2V720PConfig, Hunyuan15I2V720PConfig, Hunyuan15SR1080PConfig) from fastvideo.configs.pipelines.hyworld import HYWorldConfig -from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5T2VConfig +from fastvideo.configs.pipelines.kandinsky5 import Kandinsky5I2VConfig, Kandinsky5T2VConfig from fastvideo.configs.pipelines.lingbotworld import LingBotWorldI2V480PConfig from fastvideo.configs.pipelines.longcat import LongCatT2V480PConfig from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig @@ -501,12 +501,30 @@ def _register_configs() -> None: "kandinskylab/Kandinsky-5.0-T2V-Lite-sft-5s-Diffusers", ], model_detectors=[ - lambda path: any(token in path.lower() for token in ("kandinsky5", "kandinsky-5")), + lambda path: any(token in path.lower() for token in ("kandinsky5", "kandinsky-5")) + and "i2v" not in path.lower(), ], model_family="kandinsky5", default_preset="kandinsky5_t2v_lite_5s", ) + # Kandinsky5 I2V + register_configs( + sampling_param_cls=None, + pipeline_config_cls=Kandinsky5I2VConfig, + workload_types=(WorkloadType.I2V, ), + hf_model_paths=[ + "kandinskylab/Kandinsky-5.0-I2V-Lite-sft-5s-Diffusers", + "kandinskylab/Kandinsky-5.0-I2V-Pro-sft-5s-Diffusers", + ], + model_detectors=[ + lambda path: any(token in path.lower() for token in ("kandinsky5", "kandinsky-5")) + and "i2v" in path.lower(), + ], + model_family="kandinsky5", + default_preset="kandinsky5_i2v_lite_5s", + ) + # LongCat (T2V, I2V, VC use same config; workload varies by path) register_configs( sampling_param_cls=None,