nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[feat]: add Lucy Edit inference scaffold (#1363)

Browse Source 
Co-authored-by: Aryan Kumar <aryan5v@users.noreply.github.com>
This commit is contained in:
Aryan Kumar
2026-06-09 15:21:43 -07:00
committed by GitHub
co-authored by Aryan Kumar
parent 19a838f54f
commit 7f539424cb
12 changed files with 375 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/design/inference_schema_parity_inventory.yaml
+1
View File
@@ -108,6 +108,7 @@ surfaces:
vae_sp: generator.pipeline.preset_overrides.vae_sp
dmd_denoising_steps: generator.pipeline.preset_overrides.dmd_denoising_steps
ti2v_task: generator.pipeline.preset_overrides.ti2v_task
lucy_edit_task: generator.pipeline.preset_overrides.lucy_edit_task
boundary_ratio: generator.pipeline.preset_overrides.boundary_ratio
compatibility_only:
model_path: "Redundant with generator.model_path."
docs/inference/support_matrix.md
+4
View File
@@ -58,6 +58,7 @@ pipeline initialization and sampling.
| FastWan2.1 T2V 1.3B | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | 480P | ⭕ | ⭕ | ⭕ | ✅ | ⭕ |
| FastWan2.2 TI2V 5B Full Attn* | `FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers` | 720P | ⭕ | ⭕ | ⭕ | ✅ | ⭕ |
| Wan2.2 TI2V 5B | `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | 720P | ⭕ | ⭕ | ✅ | ⭕ | ⭕ |
| Lucy Edit Dev 5B*** | `decart-ai/Lucy-Edit-Dev` | 480P | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ |
| Wan2.2 T2V A14B | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 480P<br>720P | ❌ | ❌ | ✅ | ⭕ | ⭕ |
| Wan2.2 I2V A14B | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | 480P<br>720P | ❌ | ❌ | ✅ | ⭕ | ⭕ |
| HunyuanVideo | `hunyuanvideo-community/HunyuanVideo` | 720px1280p<br>544px960p | ❌ | ✅ | ✅ | ⭕ | ⭕ |
@@ -78,6 +79,9 @@ pipeline initialization and sampling.
**Note**: Wan2.2 TI2V 5B has some quality issues when performing I2V generation. We are working on fixing this issue.
***Lucy Edit Dev uses a non-commercial model license. FastVideo support is
focused on inference integration for video editing workflows.
`Sliding Tile Attn (Legacy Branch)` entries refer to the archived
`sta_do_not_delete` branch workflow, not active `main` inference wiring.
examples/inference/basic/basic_lucy_edit.py
+38
View File
@@ -0,0 +1,38 @@
from fastvideo import VideoGenerator
OUTPUT_PATH = "video_samples_lucy_edit"
def main():
generator = VideoGenerator.from_pretrained(
"decart-ai/Lucy-Edit-Dev",
num_gpus=1,
use_fsdp_inference=False,
dit_cpu_offload=True,
vae_cpu_offload=False,
text_encoder_cpu_offload=True,
pin_cpu_memory=True,
)
prompt = ("Change the apron and blouse to a classic clown costume: satin "
"polka-dot jumpsuit in bright primary colors, ruffled white collar, "
"oversized pom-pom buttons, white gloves, oversized red shoes, red "
"foam nose; soft window light from left, eye-level medium shot.")
video_path = "https://d2drjpuinn46lb.cloudfront.net/painter_original_edit.mp4"
generator.generate_video(
prompt,
negative_prompt="",
video_path=video_path,
output_path=OUTPUT_PATH,
save_video=True,
height=480,
width=832,
num_frames=81,
fps=24,
guidance_scale=5.0,
)
if __name__ == "__main__":
main()
fastvideo/configs/pipelines/__init__.py
+4 -4
View File
@@ -9,12 +9,12 @@ from fastvideo.configs.pipelines.matrixgame2 import MatrixGame2I2V480PConfig
from fastvideo.configs.pipelines.matrixgame3 import MatrixGame3I2V720PConfig
from fastvideo.pipelines.basic.ltx2.pipeline_configs import LTX2T2VConfig
from fastvideo.registry import get_pipeline_config_cls_from_name
from fastvideo.configs.pipelines.wan import (SelfForcingWanT2V480PConfig, WanI2V480PConfig, WanI2V720PConfig,
WanT2V480PConfig, WanT2V720PConfig)
from fastvideo.configs.pipelines.wan import (LucyEditDevConfig, SelfForcingWanT2V480PConfig, WanI2V480PConfig,
WanI2V720PConfig, WanT2V480PConfig, WanT2V720PConfig)
__all__ = [
"HunyuanConfig", "FastHunyuanConfig", "HunyuanGameCraftPipelineConfig", "PipelineConfig", "Hunyuan15T2V480PConfig",
"Hunyuan15T2V720PConfig", "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig",
"SelfForcingWanT2V480PConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig", "HYWorldConfig",
"MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name"
"SelfForcingWanT2V480PConfig", "LucyEditDevConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig",
"HYWorldConfig", "MatrixGame2I2V480PConfig", "MatrixGame3I2V720PConfig", "get_pipeline_config_cls_from_name"
]
fastvideo/configs/pipelines/base.py
+2 -1
View File
@@ -69,8 +69,9 @@ class PipelineConfig:
# DMD parameters
dmd_denoising_steps: list[int] | None = field(default=None)
# Wan2.2 TI2V parameters
# Wan2.2 task modifiers
ti2v_task: bool = False
lucy_edit_task: bool = False
boundary_ratio: float | None = None
# Compilation
fastvideo/configs/pipelines/wan.py
+138
View File
@@ -6,9 +6,11 @@ import torch
from fastvideo.configs.models import DiTConfig, EncoderConfig, VAEConfig
from fastvideo.configs.models.dits import WanVideoConfig
from fastvideo.configs.models.dits.wanvideo import WanVideoArchConfig
from fastvideo.configs.models.encoders import (BaseEncoderOutput, CLIPVisionConfig, T5Config,
WAN2_1ControlCLIPVisionConfig)
from fastvideo.configs.models.vaes import WanVAEConfig
from fastvideo.configs.models.vaes.wanvae import WanVAEArchConfig
from fastvideo.configs.pipelines.base import PipelineConfig
@@ -120,6 +122,142 @@ class Wan2_2_TI2V_5B_Config(WanT2V480PConfig):
expand_timesteps: bool = True
def __post_init__(self) -> None:
assert not (self.ti2v_task and self.lucy_edit_task)
self.vae_config.load_encoder = True
self.vae_config.load_decoder = True
self.dit_config.expand_timesteps = self.expand_timesteps
@dataclass
class LucyEditDevConfig(Wan2_2_TI2V_5B_Config):
"""Configuration for Decart Lucy Edit Dev video editing."""
dit_config: DiTConfig = field(default_factory=lambda: WanVideoConfig(arch_config=WanVideoArchConfig(
num_attention_heads=24,
in_channels=96,
out_channels=48,
ffn_dim=14336,
num_layers=30,
)))
vae_config: VAEConfig = field(default_factory=lambda: WanVAEConfig(arch_config=WanVAEArchConfig(
base_dim=160,
decoder_base_dim=256,
z_dim=48,
in_channels=12,
out_channels=12,
scale_factor_spatial=16,
patch_size=2,
is_residual=True,
clip_output=False,
latents_mean=(
-0.2289,
-0.0052,
-0.1323,
-0.2339,
-0.2799,
0.0174,
0.1838,
0.1557,
-0.1382,
0.0542,
0.2813,
0.0891,
0.1570,
-0.0098,
0.0375,
-0.1825,
-0.2246,
-0.1207,
-0.0698,
0.5109,
0.2665,
-0.2108,
-0.2158,
0.2502,
-0.2055,
-0.0322,
0.1109,
0.1567,
-0.0729,
0.0899,
-0.2799,
-0.1230,
-0.0313,
-0.1649,
0.0117,
0.0723,
-0.2839,
-0.2083,
-0.0520,
0.3748,
0.0152,
0.1957,
0.1433,
-0.2944,
0.3573,
-0.0548,
-0.1681,
-0.0667,
),
latents_std=(
0.4765,
1.0364,
0.4514,
1.1677,
0.5313,
0.4990,
0.4818,
0.5013,
0.8158,
1.0344,
0.5894,
1.0901,
0.6885,
0.6165,
0.8454,
0.4978,
0.5759,
0.3523,
0.7135,
0.6804,
0.5833,
1.4146,
0.8986,
0.5659,
0.7069,
0.5338,
0.4889,
0.4917,
0.4069,
0.4999,
0.6866,
0.4093,
0.5709,
0.6065,
0.6415,
0.4944,
0.5726,
1.2042,
0.5458,
1.6887,
0.3971,
1.0600,
0.3943,
0.5537,
0.5444,
0.4089,
0.7468,
0.7744,
),
)))
ti2v_task: bool = False
lucy_edit_task: bool = True
def __post_init__(self) -> None:
assert not (self.ti2v_task and self.lucy_edit_task)
# Lucy uses Wan2.2's enhanced 48-channel VAE latents. Denoising
# concatenates noise + video latents, matching the 96-channel
# transformer input declared above.
self.vae_config.load_encoder = True
self.vae_config.load_decoder = True
self.dit_config.expand_timesteps = self.expand_timesteps
fastvideo/pipelines/basic/wan/lucy_edit_pipeline.py
+80
View File
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
"""Lucy Edit video editing pipeline.
Lucy Edit uses a Wan2.2 5B transformer with an input video latent appended to
the noisy latent channels. The stage topology is therefore closest to Wan V2V,
but the model repo does not include CLIP image-encoder components.
"""
from fastvideo.fastvideo_args import FastVideoArgs
from fastvideo.logger import init_logger
from fastvideo.pipelines.basic.wan.wan_v2v_pipeline import WanVideoToVideoPipeline
from fastvideo.pipelines.stages import (
ConditioningStage,
DecodingStage,
DenoisingStage,
InputValidationStage,
LatentPreparationStage,
TextEncodingStage,
TimestepPreparationStage,
VideoVAEEncodingStage,
)
logger = init_logger(__name__)
class LucyEditPipeline(WanVideoToVideoPipeline):
"""FastVideo pipeline for decart-ai/Lucy-Edit-Dev."""
_required_config_modules = [
"text_encoder",
"tokenizer",
"vae",
"transformer",
"scheduler",
]
def create_pipeline_stages(self, fastvideo_args: FastVideoArgs):
self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage())
self.add_stage(
stage_name="prompt_encoding_stage",
stage=TextEncodingStage(
text_encoders=[self.get_module("text_encoder")],
tokenizers=[self.get_module("tokenizer")],
),
)
self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage())
self.add_stage(
stage_name="timestep_preparation_stage",
stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")),
)
self.add_stage(
stage_name="latent_preparation_stage",
stage=LatentPreparationStage(
scheduler=self.get_module("scheduler"),
transformer=self.get_module("transformer"),
),
)
self.add_stage(
stage_name="video_latent_preparation_stage",
stage=VideoVAEEncodingStage(vae=self.get_module("vae")),
)
self.add_stage(
stage_name="denoising_stage",
stage=DenoisingStage(
transformer=self.get_module("transformer"),
transformer_2=self.get_module("transformer_2"),
scheduler=self.get_module("scheduler"),
),
)
self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")))
EntryClass = LucyEditPipeline
fastvideo/pipelines/basic/wan/presets.py
+19
View File
@@ -268,6 +268,24 @@ FAST_WAN_2_2_TI2V_5B = InferencePreset(
},
)
LUCY_EDIT_DEV = InferencePreset(
name="lucy_edit_dev",
version=1,
model_family="wan",
description="Lucy Edit Dev 5B video editing",
workload_type="t2v",
stage_schemas=(_DENOISE_STAGE, ),
defaults={
"height": 480,
"width": 832,
"num_frames": 81,
"fps": 24,
"guidance_scale": 5.0,
"num_inference_steps": 50,
"negative_prompt": "",
},
)
# -------------------------------------------------------------------
# Self-Forcing (causal) presets
# -------------------------------------------------------------------
@@ -341,6 +359,7 @@ ALL_PRESETS = (
FAST_WAN_T2V_480P,
WAN_2_2_TI2V_5B,
FAST_WAN_2_2_TI2V_5B,
LUCY_EDIT_DEV,
SF_WAN_T2V_1_3B,
SF_WAN_2_2_T2V_A14B,
SF_WAN_2_2_I2V_A14B,
fastvideo/pipelines/stages/denoising.py
+21 -3
View File
@@ -270,6 +270,7 @@ class DenoisingStage(PipelineStage):
# Initialize lists for ODE trajectory
trajectory_timesteps: list[torch.Tensor] = []
trajectory_latents: list[torch.Tensor] = []
is_lucy_edit = fastvideo_args.pipeline_config.lucy_edit_task
# Hoisted out of the per-step loop: depends only on inputs that
# are constant across denoising steps.
@@ -291,6 +292,12 @@ class DenoisingStage(PipelineStage):
# latent_model_input. Shape is fixed by latents and is never
# written to, so we allocate once.
v2v_zero_pad = torch.zeros_like(latents) if batch.video_latent is not None else None
lucy_timestep_seq_len = None
if is_lucy_edit:
patch_size = fastvideo_args.pipeline_config.dit_config.arch_config.patch_size
assert patch_size[0] == 1, "Lucy Edit timestep expansion assumes temporal patch size 1"
lucy_timestep_seq_len = (latents.shape[2] * (latents.shape[3] // patch_size[1]) *
(latents.shape[4] // patch_size[2]))
# CFG gating / stale-uncond reuse setup (Adaptive Guidance LinearAG
# variant, Castillo et al. 2023). When envs.FASTVIDEO_CFG_GATE_STEP
@@ -371,14 +378,25 @@ class DenoisingStage(PipelineStage):
# Expand latents for V2V/I2V
latent_model_input = latents.to(target_dtype)
if batch.video_latent is not None:
latent_model_input = torch.cat([latent_model_input, batch.video_latent, v2v_zero_pad],
dim=1).to(target_dtype)
if is_lucy_edit:
latent_model_input = torch.cat(
[latent_model_input, batch.video_latent],
dim=1,
).to(target_dtype)
else:
latent_model_input = torch.cat(
[latent_model_input, batch.video_latent, v2v_zero_pad],
dim=1,
).to(target_dtype)
elif batch.image_latent is not None:
assert not fastvideo_args.pipeline_config.ti2v_task, "image latents should not be provided for TI2V task"
latent_model_input = torch.cat([latent_model_input, batch.image_latent], dim=1).to(target_dtype)
assert not torch.isnan(latent_model_input).any(), "latent_model_input contains nan"
if fastvideo_args.pipeline_config.ti2v_task and batch.pil_image is not None:
if is_lucy_edit:
assert lucy_timestep_seq_len is not None
t_expand = t.repeat(latent_model_input.shape[0], lucy_timestep_seq_len)
elif fastvideo_args.pipeline_config.ti2v_task and batch.pil_image is not None:
timestep = torch.stack([t]).to(get_local_torch_device())
temp_ts = (mask2[0][0][:, ::2, ::2] * timestep).flatten()
temp_ts = torch.cat([temp_ts, temp_ts.new_ones(seq_len - temp_ts.size(0)) * timestep])
fastvideo/pipelines/stages/image_encoding.py
+4 -3
View File
@@ -629,9 +629,10 @@ class VideoVAEEncodingStage(ImageVAEEncodingStage):
encoder_output = self.vae.encode(video_condition)
generator = batch.generator
if generator is None:
raise ValueError("Generator must be provided")
latent_condition = self.retrieve_latents(encoder_output, generator)
sample_mode = "argmax" if fastvideo_args.pipeline_config.lucy_edit_task else "sample"
if sample_mode == "sample" and generator is None:
raise ValueError("Generator must be provided for sampled video VAE encoding")
latent_condition = self.retrieve_latents(encoder_output, generator, sample_mode=sample_mode)
if (hasattr(self.vae, "shift_factor") and self.vae.shift_factor is not None):
if isinstance(self.vae.shift_factor, torch.Tensor):
fastvideo/registry.py
+13
View File
@@ -44,6 +44,7 @@ from fastvideo.configs.pipelines.turbodiffusion import (
from fastvideo.configs.pipelines.wan import (
FastWan2_1_T2V_480P_Config,
FastWan2_2_TI2V_5B_Config,
LucyEditDevConfig,
SelfForcingWan2_2_T2V480PConfig,
SelfForcingWanT2V480PConfig,
WANV2VConfig,
@@ -781,6 +782,18 @@ def _register_configs() -> None:
model_family="wan",
default_preset="fast_wan_2_2_ti2v_5b",
)
register_configs(
sampling_param_cls=None,
pipeline_config_cls=LucyEditDevConfig,
workload_types=(),
hf_model_paths=[
"decart-ai/Lucy-Edit-Dev",
"decart-ai/Lucy-Edit-1.1-Dev",
],
model_detectors=[lambda path: "lucy-edit" in path.lower()],
model_family="wan",
default_preset="lucy_edit_dev",
)
register_configs(
sampling_param_cls=None,
pipeline_config_cls=Wan2_2_T2V_A14B_Config,
fastvideo/tests/api/test_lucy_edit.py
+51
View File
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
from fastvideo.api.presets import get_preset
from fastvideo.api.sampling_param import SamplingParam
from fastvideo.configs.pipelines.wan import LucyEditDevConfig
from fastvideo.fastvideo_args import WorkloadType
from fastvideo.pipelines.basic.wan.lucy_edit_pipeline import LucyEditPipeline
from fastvideo.pipelines.pipeline_registry import PipelineType, get_pipeline_registry
from fastvideo.registry import get_default_preset, get_pipeline_config_cls_from_name
def test_lucy_edit_registry_and_preset() -> None:
import fastvideo.registry # noqa: F401
preset = get_preset("lucy_edit_dev", "wan")
assert preset.model_family == "wan"
assert preset.defaults["height"] == 480
assert preset.defaults["width"] == 832
assert preset.defaults["num_frames"] == 81
config = LucyEditDevConfig()
assert config.lucy_edit_task is True
assert config.ti2v_task is False
assert config.dit_config.arch_config.out_channels == 48
assert config.dit_config.arch_config.in_channels == 96
assert config.vae_config.arch_config.z_dim == 48
assert config.dit_config.arch_config.in_channels == config.vae_config.arch_config.z_dim * 2
assert get_default_preset("decart-ai/Lucy-Edit-Dev") == "lucy_edit_dev"
assert get_default_preset("decart-ai/Lucy-Edit-1.1-Dev") == "lucy_edit_dev"
assert get_pipeline_config_cls_from_name("decart-ai/Lucy-Edit-Dev") is LucyEditDevConfig
assert get_pipeline_config_cls_from_name("decart-ai/Lucy-Edit-1.1-Dev") is LucyEditDevConfig
sampling_param = SamplingParam.from_pretrained("decart-ai/Lucy-Edit-Dev")
assert sampling_param.height == 480
assert sampling_param.width == 832
assert sampling_param.num_frames == 81
assert sampling_param.fps == 24
assert sampling_param.guidance_scale == 5.0
assert sampling_param.negative_prompt == ""
sampling_param_1_1 = SamplingParam.from_pretrained("decart-ai/Lucy-Edit-1.1-Dev")
assert sampling_param_1_1.height == 480
assert sampling_param_1_1.width == 832
assert sampling_param_1_1.num_frames == 81
assert sampling_param_1_1.fps == 24
assert sampling_param_1_1.guidance_scale == 5.0
assert sampling_param_1_1.negative_prompt == ""
# FastVideo has no V2V workload enum today; model_index dispatches Lucy by pipeline class name.
registry = get_pipeline_registry(PipelineType.BASIC)
assert registry.resolve_pipeline_cls("LucyEditPipeline", PipelineType.BASIC, WorkloadType.T2V) is LucyEditPipeline