endman100-ComfyUI_SGLDiffusion_Fix/executors/qwen_image.py

"""
QwenImage executor for SGLang Diffusion ComfyUI integration.
"""

import torch

try:
    from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
    from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
except ImportError:
    print(
        "Error: sglang.multimodal_gen is not installed. Please install it using 'pip install sglang[diffusion]'"
    )

import comfy.ldm.common_dit

from .base import SGLDiffusionExecutor


class QwenImageExecutor(SGLDiffusionExecutor):
    """Executor for QwenImage models in ComfyUI."""

    def __init__(self, generator, model_path, model, config):
        super().__init__(generator, model_path, model, config)
        self.patch_size = 2

    def _pack_latents(self, x):
        """Process hidden states for QwenImage model."""
        bs, c, t, h, w = x.shape
        patch_size = self.patch_size
        latents = comfy.ldm.common_dit.pad_to_patch_size(
            x, (1, self.patch_size, self.patch_size)
        )
        orig_shape = latents.shape
        latents = latents.view(
            orig_shape[0],
            orig_shape[1],
            orig_shape[-3],
            orig_shape[-2] // 2,
            2,
            orig_shape[-1] // 2,
            2,
        )
        latents = latents.permute(0, 2, 3, 5, 1, 4, 6)
        latents = latents.reshape(
            orig_shape[0],
            orig_shape[-3] * (orig_shape[-2] // 2) * (orig_shape[-1] // 2),
            orig_shape[1] * 4,
        )
        return latents, orig_shape

    def _unpack_latents(self, latents, num_embeds, orig_shape, x):
        """Unpack hidden states from packed format to standard format."""
        latents = latents[:, :num_embeds].view(
            orig_shape[0],
            orig_shape[-3],
            orig_shape[-2] // 2,
            orig_shape[-1] // 2,
            orig_shape[1],
            2,
            2,
        )
        latents = latents.permute(0, 4, 1, 2, 5, 3, 6)
        latents = latents.reshape(orig_shape)[:, :, :, : x.shape[-2], : x.shape[-1]]
        return latents

    def forward(self, x, timestep, context, **kwargs):
        """Forward pass for QwenImage model."""
        latents, orig_shape = self._pack_latents(x)
        num_embeds = latents.shape[1]
        height = orig_shape[-2] * 8
        width = orig_shape[-1] * 8

        sampling_params = SamplingParams.from_user_sampling_params_args(
            self.model_path,
            server_args=self.generator.server_args,
            prompt=" ",
            guidance_scale=1.0,
            height=height,
            width=width,
            num_frames=1,
            num_inference_steps=1,
            save_output=False,
            suppress_logs=self.should_suppress_logs(timestep),
        )

        # Prepare request (converts SamplingParams to Req)
        req = prepare_request(
            server_args=self.generator.server_args,
            sampling_params=sampling_params,
        )
        # Set ComfyUI-specific inputs directly on the Req object
        req.latents = latents
        req.timesteps = timestep * 1000.0
        req.prompt_embeds = [context]
        req.raw_latent_shape = torch.tensor(latents.shape, dtype=torch.long)
        req.do_classifier_free_guidance = False
        req.generator = [
            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
        ]

        output_batch = self.generator._send_to_scheduler_and_wait_for_response([req])
        noise_pred = output_batch.noise_pred

        return self._unpack_latents(noise_pred, num_embeds, orig_shape, x)


class QwenImageEditExecutor(QwenImageExecutor):
    """Executor for QwenImageEdit models in ComfyUI."""

    def __init__(self, generator, model_path, model, config):
        super().__init__(generator, model_path, model, config)

    def forward(
        self,
        x,
        timestep,
        context,
        attention_mask=None,
        ref_latents=None,
        additional_t_cond=None,
        transformer_options={},
        **kwargs
    ):
        """Forward pass for QwenImageEdit model."""
        latents, orig_shape = self._pack_latents(x)
        num_embeds = latents.shape[1]
        height = orig_shape[-2] * 8
        width = orig_shape[-1] * 8

        # Prepare vae_image_sizes for the condition image (ref_latents)
        vae_image_sizes = []
        pack_ref_latents = None

        # TODO: sgld now don't support multiple condition images, so we only support one condition image for now.
        if ref_latents is not None and len(ref_latents) > 0:
            pack_ref_latents, orig_ref_shape = self._pack_latents(ref_latents[0])
            vae_image_sizes = [(orig_ref_shape[-1], orig_ref_shape[-2])]

        sampling_params = SamplingParams.from_user_sampling_params_args(
            self.model_path,
            server_args=self.generator.server_args,
            prompt=" ",
            guidance_scale=1.0,
            image_path="",
            height=height,
            width=width,
            num_frames=1,
            num_inference_steps=1,
            save_output=False,
            suppress_logs=self.should_suppress_logs(timestep),
        )

        # Prepare request (converts SamplingParams to Req)
        req = prepare_request(
            server_args=self.generator.server_args,
            sampling_params=sampling_params,
        )
        # Set ComfyUI-specific inputs directly on the Req object
        req.latents = latents
        req.image_latent = pack_ref_latents
        req.timesteps = timestep * 1000.0
        req.vae_image_sizes = vae_image_sizes
        req.prompt_embeds = [context]
        req.raw_latent_shape = torch.tensor(latents.shape, dtype=torch.long)
        req.do_classifier_free_guidance = False
        req.generator = [
            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
        ]

        output_batch = self.generator._send_to_scheduler_and_wait_for_response([req])
        noise_pred = output_batch.noise_pred

        return self._unpack_latents(noise_pred, num_embeds, orig_shape, x)