kijai-ComfyUI-WanVideoWrapper/Ovi/nodes_ovi.py

import torch
import torch.nn as nn
import folder_paths
import os

from .mel_converter import get_mel_converter
from .vae.autoencoder import AutoEncoderModule
from .vae.distributions import DiagonalGaussianDistribution
import torchaudio

from ..utils import log

from comfy import model_management as mm
device = mm.get_torch_device()
offload_device = mm.unet_offload_device()

class FeaturesUtils(nn.Module):

    def __init__(
        self,
        *,
        tod_vae_ckpt: str,
        bigvgan_vocoder_ckpt = None,
        mode=['16k', '44k'],
        need_vae_encoder: bool = True,
    ):
        super().__init__()

        self.mel_converter = get_mel_converter(mode)
        self.tod = AutoEncoderModule(vae_ckpt_path=tod_vae_ckpt,
                                        vocoder_ckpt_path=bigvgan_vocoder_ckpt,
                                        mode=mode,
                                        need_vae_encoder=need_vae_encoder)

    def encode_audio(self, x) -> DiagonalGaussianDistribution:
        assert self.tod is not None, 'VAE is not loaded'
        # x: (B * L)
        mel = self.mel_converter(x)
        dist = self.tod.encode(mel)

        return dist

    def vocode(self, mel: torch.Tensor) -> torch.Tensor:
        assert self.tod is not None, 'VAE is not loaded'
        return self.tod.vocode(mel)

    def decode(self, z: torch.Tensor) -> torch.Tensor:
        assert self.tod is not None, 'VAE is not loaded'
        return self.tod.decode(z)

    @property
    def device(self):
        return next(self.parameters()).device

    @property
    def dtype(self):
        return next(self.parameters()).dtype

    def wrapped_decode(self, z):
        with torch.amp.autocast('cuda', dtype=self.dtype):
            mel_decoded = self.decode(z)
            audio = self.vocode(mel_decoded)

            return audio

    def wrapped_encode(self, audio):
        with torch.amp.autocast('cuda', dtype=self.dtype):
            dist = self.encode_audio(audio)

            return dist.mean

if not "mmaudio" in folder_paths.folder_names_and_paths:
    folder_paths.add_model_folder_path("mmaudio", os.path.join(folder_paths.models_dir, "mmaudio"))

class OviMMAudioVAELoader:
    """Loads MMAudio VAE for audio encoding/decoding in Ovi"""
    @classmethod
    def INPUT_TYPES(s):
        s.vae_files = folder_paths.get_filename_list("vae")
        s.mmaudio_files = folder_paths.get_filename_list("mmaudio")
        s.all_files = s.vae_files + s.mmaudio_files

        return {
            "required": {
                "vae": (s.all_files, {"tooltip": "MMAudio VAE 16k (v1-16.pth) model from models/vae or models/mmaudio"}),
                "vocoder": (s.all_files, {"tooltip": "BigVGAN vocoder (best_netG.pt) from models/vae or models/mmaudio"}),
                "precision": (["bf16", "fp16", "fp32"], {"default": "bf16"}),
            }
        }

    RETURN_TYPES = ("MMAUDIOVAE",)
    RETURN_NAMES = ("mmaudio_vae",)
    FUNCTION = "loadmodel"
    CATEGORY = "WanVideoWrapper/Ovi"
    DESCRIPTION = "Loads MMAudio VAE for Ovi audio generation"

    def loadmodel(self, vae, vocoder, precision):
        dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision]

        vae_path = folder_paths.get_full_path("vae", vae) if vae in self.vae_files else folder_paths.get_full_path("mmaudio", vae)
        vocoder_path = folder_paths.get_full_path("vae", vocoder) if vocoder in self.vae_files else folder_paths.get_full_path("mmaudio", vocoder)

        vae = FeaturesUtils(
            tod_vae_ckpt=vae_path,
            bigvgan_vocoder_ckpt=vocoder_path,
            mode='16k',
            need_vae_encoder=True
        )

        vae.to(device=offload_device, dtype=dtype)
        vae.eval()

        return (vae,)

class WanVideoDecodeOviAudio:
    @classmethod
    def INPUT_TYPES(s):
        return {"required": {
                    "mmaudio_vae": ("MMAUDIOVAE",),
                    "samples": ("LATENT",),
                }
        }

    RETURN_TYPES = ("AUDIO",)
    RETURN_NAMES = ("audio",)
    FUNCTION = "decode"
    CATEGORY = "WanVideoWrapper/Ovi"

    def decode(self, mmaudio_vae, samples):
        mm.soft_empty_cache()
        audio_latents = samples.get("latent_ovi_audio", None)
        if audio_latents is None:
            raise ValueError("No Ovi audio latents found in input samples")

        mmaudio_vae.to(device)

        waveform = mmaudio_vae.wrapped_decode(audio_latents.to(device=device, dtype=mmaudio_vae.dtype))
        audio = {"waveform": waveform.cpu().float(), "sample_rate": 16000}

        mmaudio_vae.to(offload_device)
        mm.soft_empty_cache()

        return (audio,)

class WanVideoEncodeOviAudio:
    @classmethod
    def INPUT_TYPES(s):
        return {"required": {
                    "mmaudio_vae": ("MMAUDIOVAE",),
                    "audio": ("AUDIO",),
                }
        }

    RETURN_TYPES = ("LATENT",)
    RETURN_NAMES = ("samples",)
    FUNCTION = "decode"
    CATEGORY = "WanVideoWrapper/Ovi"

    def decode(self, mmaudio_vae, audio):

        mmaudio_vae.to(device)

        waveform = audio.get("waveform", None)
        sample_rate = audio.get("sample_rate", None)
        if sample_rate != 16000:
            waveform = torchaudio.functional.resample(waveform, sample_rate, 16000)
        waveform = waveform.to(device=device, dtype=mmaudio_vae.dtype)[0][0].unsqueeze(0)

        samples = mmaudio_vae.wrapped_encode(waveform)

        mmaudio_vae.to(offload_device)
        mm.soft_empty_cache()

        return ({"latent_ovi_audio": samples},)


class WanVideoAddOviAudioToLatents:
    @classmethod
    def INPUT_TYPES(s):
        return {"required": {
                    "original_samples": ("LATENT",),
                    "audio_samples": ("LATENT",),
                }
        }

    RETURN_TYPES = ("LATENT",)
    RETURN_NAMES = ("samples",)
    FUNCTION = "decode"
    CATEGORY = "WanVideoWrapper/Ovi"

    def decode(self, original_samples, audio_samples):
        samples = original_samples.copy()
        samples.update(audio_samples)

        return (samples,)
    
class WanVideoEmptyMMAudioLatents:
    @classmethod
    def INPUT_TYPES(s):
        return {"required": {
                    "length": ("INT", {"default": 157, "min": 1, "max": 10000, "step": 1, "tooltip": "Length of the audio latent sequence"}),
                }
        }

    RETURN_TYPES = ("LATENT",)
    RETURN_NAMES = ("samples",)
    FUNCTION = "decode"
    CATEGORY = "WanVideoWrapper/Ovi"

    def decode(self, length):
        audio_latents = torch.zeros((length, 20), device=torch.device("cpu"), dtype=torch.float32)  # 1, l c -> l, c

        return ({"latent_ovi_audio": audio_latents},)


class WanVideoOviCFG:
    @classmethod
    def INPUT_TYPES(s):
        return {"required": {
            "original_text_embeds": ("WANVIDEOTEXTEMBEDS",),
            "ovi_audio_cfg": ("FLOAT", {"default": 3.0, "min": 0.0, "max": 100.0, "step": 0.01}),
            },
            "optional": {
                "ovi_negative_text_embeds": ("WANVIDEOTEXTEMBEDS",),
            }
        }

    RETURN_TYPES = ("WANVIDEOTEXTEMBEDS", )
    RETURN_NAMES = ("text_embeds",)
    FUNCTION = "process"
    CATEGORY = "WanVideoWrapper/Ovi"
    DESCRIPTION = "Adds Ovi negative text embeddings and audio CFG scale to the text embeddings dictionary"

    def process(self, original_text_embeds, ovi_audio_cfg, ovi_negative_text_embeds=None):
        negative_text_embeds = None
        if ovi_negative_text_embeds is not None:
            negative_text_embeds = ovi_negative_text_embeds.get("prompt_embeds", None)
        if negative_text_embeds is None:
            negative_text_embeds = original_text_embeds["prompt_embeds"]
            log.info("WanVideoOviCFG: Ovi negative text embeddings not provided, using original prompt embeddings as negative embeddings")
        else:
            log.info("WanVideoOviCFG: Using provided Ovi audio negative text embeddings")
        log.info("WanVideoOviCFG: negative text embedding shape: {}".format(negative_text_embeds[0].shape))

        prompt_embeds_dict_copy = original_text_embeds.copy()
        prompt_embeds_dict_copy.update({
                "ovi_negative_prompt_embeds": negative_text_embeds,
                "ovi_audio_cfg": ovi_audio_cfg,
            })
        return (prompt_embeds_dict_copy,)

NODE_CLASS_MAPPINGS = {
    "OviMMAudioVAELoader": OviMMAudioVAELoader,
    "WanVideoDecodeOviAudio": WanVideoDecodeOviAudio,
    "WanVideoEncodeOviAudio": WanVideoEncodeOviAudio,
    "WanVideoOviCFG": WanVideoOviCFG,
    "WanVideoAddOviAudioToLatents": WanVideoAddOviAudioToLatents,
    "WanVideoEmptyMMAudioLatents": WanVideoEmptyMMAudioLatents,
}
NODE_DISPLAY_NAME_MAPPINGS = {
    "OviMMAudioVAELoader": "Ovi MMAudio VAE Loader",
    "WanVideoDecodeOviAudio": "WanVideo Decode Ovi Audio",
    "WanVideoEncodeOviAudio": "WanVideo Encode Ovi Audio",
    "WanVideoOviCFG": "WanVideo Ovi CFG",
    "WanVideoAddOviAudioToLatents": "WanVideo Add MMAudio To Latents",
    "WanVideoEmptyMMAudioLatents": "WanVideo Empty MMAudio Latents",
}