aszc-dev-ComfyUI-CoreMLSuite/coreml_suite/converter.py

import gc
import os
import shutil
import time
from typing import Union

import coremltools as ct
import numpy as np
import python_coreml_stable_diffusion.unet
import torch
from diffusers import (
    StableDiffusionPipeline,
    LatentConsistencyModelPipeline,
    StableDiffusionXLPipeline,
)
from python_coreml_stable_diffusion.unet import (
    UNet2DConditionModel,
    UNet2DConditionModelXL,
    AttentionImplementations,
)

from coreml_suite.config import ModelVersion
from coreml_suite.lcm.unet import UNet2DConditionModelLCM
from coreml_suite.logger import logger
from folder_paths import get_folder_paths


class StableDiffusionLCMPipeline(LatentConsistencyModelPipeline):
    pass


MODEL_TYPE_TO_UNET_CLS = {
    ModelVersion.SD15: UNet2DConditionModel,
    ModelVersion.SDXL: UNet2DConditionModelXL,
    ModelVersion.LCM: UNet2DConditionModelLCM,
}

MODEL_TYPE_TO_PIPE_CLS = {
    ModelVersion.SD15: StableDiffusionPipeline,
    ModelVersion.SDXL: StableDiffusionXLPipeline,
    ModelVersion.LCM: StableDiffusionLCMPipeline,
}


def get_unet(model_type: ModelVersion, ref_pipe):
    ref_unet = ref_pipe.unet

    unet_cls = MODEL_TYPE_TO_UNET_CLS[model_type]
    cml_unet = unet_cls.from_config(ref_unet.config).eval()
    cml_unet.load_state_dict(ref_unet.state_dict(), strict=False)

    return cml_unet


def get_encoder_hidden_states_shape(ref_pipe, batch_size):
    text_encoder = (
        ref_pipe.text_encoder_2
        if hasattr(ref_pipe, "text_encoder_2")
        else ref_pipe.text_encoder
    )

    text_token_sequence_length = text_encoder.config.max_position_embeddings
    hidden_size = (text_encoder.config.hidden_size,)

    encoder_hidden_states_shape = (
        batch_size,
        ref_pipe.unet.config.cross_attention_dim or hidden_size,
        1,
        text_token_sequence_length,
    )

    return encoder_hidden_states_shape


def get_coreml_inputs(sample_inputs):
    coreml_sample_unet_inputs = {
        k: v.numpy().astype(np.float16) for k, v in sample_inputs.items()
    }
    return [
        ct.TensorType(
            name=k,
            shape=v.shape,
            dtype=v.numpy().dtype if isinstance(v, torch.Tensor) else v.dtype,
        )
        for k, v in coreml_sample_unet_inputs.items()
    ]


def load_coreml_model(out_path):
    logger.info(f"Loading model from {out_path}")

    start = time.time()
    coreml_model = ct.models.MLModel(out_path)
    logger.info(f"Loading {out_path} took {time.time() - start:.1f} seconds")

    return coreml_model


def convert_to_coreml(
    submodule_name, torchscript_module, sample_inputs, output_names, out_path
):
    if os.path.exists(out_path):
        logger.info(f"Skipping export because {out_path} already exists")
        coreml_model = load_coreml_model(out_path)
    else:
        logger.info(f"Converting {submodule_name} to CoreML..")
        coreml_model = ct.convert(
            torchscript_module,
            convert_to="mlprogram",
            minimum_deployment_target=ct.target.macOS13,
            inputs=sample_inputs,
            outputs=[
                ct.TensorType(name=name, dtype=np.float32) for name in output_names
            ],
            skip_model_load=True,
        )

        del torchscript_module
        gc.collect()

    return coreml_model


def get_out_path(submodule_name, model_name):
    fname = f"{model_name}_{submodule_name}.mlpackage"
    unet_path = get_folder_paths(submodule_name)[0]
    out_path = os.path.join(unet_path, fname)
    return out_path


def compile_coreml_model(source_model_path, output_dir, final_name):
    """Compiles Core ML models using the coremlcompiler utility from Xcode toolchain"""
    target_path = os.path.join(output_dir, f"{final_name}.mlmodelc")
    if os.path.exists(target_path):
        logger.warning(f"Found existing compiled model at {target_path}! Skipping..")
        return target_path

    logger.info(f"Compiling {source_model_path}")
    source_model_name = os.path.basename(os.path.splitext(source_model_path)[0])

    os.system(f"xcrun coremlcompiler compile {source_model_path} {output_dir}")
    compiled_output = os.path.join(output_dir, f"{source_model_name}.mlmodelc")
    shutil.move(compiled_output, target_path)

    return target_path


def get_sample_input(batch_size, encoder_hidden_states_shape, sample_shape, scheduler):
    sample_unet_inputs = dict(
        [
            ("sample", torch.rand(*sample_shape)),
            (
                "timestep",
                torch.tensor([scheduler.timesteps[0].item()] * batch_size).to(
                    torch.float32
                ),
            ),
            ("encoder_hidden_states", torch.rand(*encoder_hidden_states_shape)),
        ]
    )
    return sample_unet_inputs


def lcm_inputs(sample_unet_inputs):
    batch_size = sample_unet_inputs["sample"].shape[0]
    return {"timestep_cond": torch.randn(batch_size, 256).to(torch.float32)}


def sdxl_inputs(sample_unet_inputs, ref_pipe):
    sample_shape = sample_unet_inputs["sample"].shape
    batch_size = sample_shape[0]
    h = sample_shape[2] * 8
    w = sample_shape[3] * 8
    original_size = (h, w)
    crops_coords_top_left = (0, 0)

    is_refiner = (
        hasattr(ref_pipe.config, "requires_aesthetics_score")
        and ref_pipe.config.requires_aesthetics_score
    )

    if is_refiner:
        aesthetic_score = (6.0,)
        time_ids_list = list(original_size + crops_coords_top_left + aesthetic_score)
    else:
        target_size = (h, w)
        time_ids_list = list(original_size + crops_coords_top_left + target_size)

    time_ids = torch.tensor(time_ids_list).repeat(batch_size, 1).to(torch.int64)
    text_embeds_shape = (batch_size, ref_pipe.text_encoder_2.config.hidden_size)

    return {
        "time_ids": time_ids,
        "text_embeds": torch.randn(*text_embeds_shape).to(torch.float32),
    }


def get_inputs_spec(inputs):
    inputs_spec = {k: (v.shape, v.dtype) for k, v in inputs.items()}
    return inputs_spec


def add_cnet_support(sample_shape, reference_unet):
    from python_coreml_stable_diffusion.unet import calculate_conv2d_output_shape

    additional_residuals_shapes = []

    batch_size = sample_shape[0]
    h, w = sample_shape[2:]

    # conv_in
    out_h, out_w = calculate_conv2d_output_shape(
        h,
        w,
        reference_unet.conv_in,
    )
    additional_residuals_shapes.append(
        (batch_size, reference_unet.conv_in.out_channels, out_h, out_w)
    )

    # down_blocks
    for down_block in reference_unet.down_blocks:
        additional_residuals_shapes += [
            (batch_size, resnet.out_channels, out_h, out_w)
            for resnet in down_block.resnets
        ]
        if hasattr(down_block, "downsamplers") and down_block.downsamplers is not None:
            for downsampler in down_block.downsamplers:
                out_h, out_w = calculate_conv2d_output_shape(
                    out_h, out_w, downsampler.conv
                )
            additional_residuals_shapes.append(
                (
                    batch_size,
                    down_block.downsamplers[-1].conv.out_channels,
                    out_h,
                    out_w,
                )
            )

    # mid_block
    additional_residuals_shapes.append(
        (batch_size, reference_unet.mid_block.resnets[-1].out_channels, out_h, out_w)
    )

    additional_inputs = {}
    for i, shape in enumerate(additional_residuals_shapes):
        sample_residual_input = torch.rand(*shape)
        additional_inputs[f"additional_residual_{i}"] = sample_residual_input

    return additional_inputs


def convert_unet(
    ref_pipe,
    model_version: ModelVersion,
    unet_out_path: str,
    batch_size: int = 1,
    sample_size: tuple[int, int] = (64, 64),
    controlnet_support: bool = False,
    quantize_nbits: str = "none",
):
    coreml_unet = get_unet(model_version, ref_pipe)
    ref_unet = ref_pipe.unet

    sample_shape = (
        batch_size,  # B
        ref_unet.config.in_channels,  # C
        sample_size[0],  # H
        sample_size[1],  # W
    )

    encoder_hidden_states_shape = get_encoder_hidden_states_shape(ref_pipe, batch_size)

    scheduler = ref_pipe.scheduler
    scheduler.set_timesteps(50)

    sample_inputs = get_sample_input(
        batch_size, encoder_hidden_states_shape, sample_shape, scheduler
    )

    if model_version == ModelVersion.LCM:
        sample_inputs |= lcm_inputs(sample_inputs)

    if model_version == ModelVersion.SDXL:
        sample_inputs |= sdxl_inputs(sample_inputs, ref_pipe)

    if controlnet_support:
        sample_inputs |= add_cnet_support(sample_shape, ref_unet)

    sample_inputs_spec = get_inputs_spec(sample_inputs)

    logger.info(f"Sample UNet inputs spec: {sample_inputs_spec}")
    logger.info("JIT tracing..")
    traced_unet = torch.jit.trace(
        coreml_unet, example_inputs=list(sample_inputs.values())
    )
    logger.info("Done.")

    coreml_sample_inputs = get_coreml_inputs(sample_inputs)

    coreml_unet = convert_to_coreml(
        "unet", traced_unet, coreml_sample_inputs, ["noise_pred"], unet_out_path
    )

    del traced_unet
    gc.collect()

    if quantize_nbits != "none":
        # Phase 6: opt-in k-means weight palettization. Default path
        # (quantize_nbits="none") is byte-for-byte unchanged from Phase 5.
        from coremltools.optimize.coreml import (
            OpPalettizerConfig,
            OptimizationConfig,
            palettize_weights,
        )

        nbits = int(quantize_nbits)
        logger.info(f"Palettizing UNet weights to {nbits}-bit (kmeans)..")
        t0 = time.time()
        cfg = OptimizationConfig(
            global_config=OpPalettizerConfig(mode="kmeans", nbits=nbits)
        )
        coreml_unet = palettize_weights(coreml_unet, config=cfg)
        logger.info(f"Palettization took {time.time() - t0:.1f}s")

    coreml_unet.save(unet_out_path)
    logger.info(f"Saved unet into {unet_out_path}")


def convert(
    ckpt_path: str,
    model_version: ModelVersion,
    unet_out_path: str,
    batch_size: int = 1,
    sample_size: tuple[int, int] = (64, 64),
    controlnet_support: bool = False,
    lora_weights: list[tuple[Union[str, os.PathLike], float]] = None,
    attn_impl: str = AttentionImplementations.SPLIT_EINSUM.name,
    config_path: str = None,
    quantize_nbits: str = "none",
):
    if os.path.exists(unet_out_path):
        logger.info(f"Found existing model at {unet_out_path}! Skipping..")
        return

    python_coreml_stable_diffusion.unet.ATTENTION_IMPLEMENTATION_IN_EFFECT = (
        AttentionImplementations(attn_impl)
    )

    ref_pipe = get_pipeline(ckpt_path, config_path, model_version)

    for i, lora_weight in enumerate(lora_weights or []):
        lora_path, strength = lora_weight
        adapter_name = f"lora_{i}"
        ref_pipe.load_lora_weights(lora_path, adapter_name=adapter_name)
        ref_pipe.set_adapters([adapter_name], adapter_weights=[strength])
        ref_pipe.fuse_lora()

    convert_unet(
        ref_pipe,
        model_version,
        unet_out_path,
        batch_size,
        sample_size,
        controlnet_support,
        quantize_nbits=quantize_nbits,
    )


def get_pipeline(ckpt_path, config_path, model_version):
    pipe_cls = MODEL_TYPE_TO_PIPE_CLS[model_version]
    ref_pipe = pipe_cls.from_single_file(ckpt_path, original_config_file=config_path)
    return ref_pipe


def compile_model(out_path, out_name, submodule_name):
    # Compile the model
    target_path = compile_coreml_model(
        out_path, get_folder_paths(submodule_name)[0], f"{out_name}_{submodule_name}"
    )
    logger.info(f"Compiled {out_path} to {target_path}")
    return target_path