kunieone-ComfyUI_alkaid/loader.py

import os
import re
import comfy.utils
import folder_paths
import comfy.model_management
from comfy.clip_vision import load as load_clip_vision
import torch
from .tools.estimate3dv import Estimate3DV
from .tools.faceskin import FaceSkin
from .ipadapter.init_proj import IDAdapter, StyleAdapter, InstantID


INSIGHTFACE_DIR = os.path.join(folder_paths.models_dir, "insightface")
ALKAID_DIR = os.path.join(folder_paths.models_dir, "alkaid")
IPADAPTER_DIR = os.path.join(folder_paths.models_dir, "ipadapter")

def insightface_loader(provider, name):
    try:
        from insightface.app import FaceAnalysis
    except ImportError as e:
        raise Exception(e)
    
    model = FaceAnalysis(name=name, root=INSIGHTFACE_DIR, providers=[provider + 'ExecutionProvider',], allowed_modules=['detection', 'recognition', ])
    model.prepare(ctx_id=0, det_size=(640, 640))
    return model



class AlkaidLoader:
    def __init__(self) -> None:
        self.insightface = None
        self.insightface_b = None
        self.estimate3d = None
        self.clip_vision = None

    @classmethod
    def INPUT_TYPES(s):
        return {"required": {
            },
        }

    RETURN_TYPES = ("ALKAIDLOADER", )
    RETURN_NAMES = ("alkaid_loader", )
    FUNCTION = "load_models"
    CATEGORY = "Alkaid/Loader"

    @staticmethod
    def get_clipvision():
        clipvision_list = folder_paths.get_filename_list("clip_vision")


        pattern = '(ViT.H.14.*s32B.b79K.(bin|safetensors))'
        clipvision_file = [e for e in clipvision_list if re.search(pattern, e, re.IGNORECASE)]

        clipvision_file = folder_paths.get_full_path("clip_vision", clipvision_file[0]) if clipvision_file else None
        clip_vision = comfy.clip_vision.load(clipvision_file)
        comfy.model_management.load_model_gpu(clip_vision.patcher)
        return clip_vision


    def load_models(self,):
        device = comfy.model_management.get_torch_device()
        provider = "CUDA"
        self.insightface = insightface_loader(provider, name='antelopev2')
        self.insightface_b = insightface_loader(provider, name='buffalo_l')
        estimate3d = Estimate3DV(checkpoint=os.path.join(ALKAID_DIR, 'bfm_estimate.pth')).eval()
        self.estimate3d = estimate3d.to(device=device, )
        self.clip_vision = self.get_clipvision()
        return ({'insightface': self.insightface,
                 'insightface_b': self.insightface_b,
                 'estimate3d': self.estimate3d,
                 'clipvision': self.clip_vision
                 },)
    

# class AdapterFaceLoader_:

#     @classmethod
#     def INPUT_TYPES(s):
#         return {"required": {
#         }}

#     RETURN_TYPES = ("ADAPTERFACELOADER",)
#     RETURN_NAMES = ("adapterface_loader", )

#     FUNCTION = "load_model"
#     CATEGORY = "Alkaid/Loader"

#     def load_model(self):
#         device = comfy.model_management.get_torch_device()
#         dtype = torch.float16 if comfy.model_management.should_use_fp16() else torch.bfloat16 if comfy.model_management.should_use_bf16() else torch.float32

#         ckpt_path = os.path.join(ALKAID_DIR, "idadapter.bin")
#         model = comfy.utils.load_torch_file(ckpt_path, safe_load=True)


#         return (model, )
    

class AdapterFaceLoader:

    @classmethod
    def INPUT_TYPES(s):
        return {"required": {
        }}

    RETURN_TYPES = ("ADAPTERFACELOADER",)
    RETURN_NAMES = ("adapterface_loader", )

    FUNCTION = "load_model"
    CATEGORY = "Alkaid/Loader"

    def load_model(self):
        device = comfy.model_management.get_torch_device()
        dtype = torch.float16 if comfy.model_management.should_use_fp16() else torch.bfloat16 if comfy.model_management.should_use_bf16() else torch.float32

        ckpt_path = os.path.join(ALKAID_DIR, "idadapter.bin")
        model = comfy.utils.load_torch_file(ckpt_path, safe_load=True)
        output_cross_attention_dim = 2048

        idadapter = IDAdapter(
            model['ipadapter'],
            cross_attention_dim=2048,
            output_cross_attention_dim=output_cross_attention_dim,
            clip_embeddings_dim=512,
            clip_extra_context_tokens=16,
        ).to(dtype=dtype)

        instantid = InstantID(
            model['instantid'],
            cross_attention_dim=1280,
            output_cross_attention_dim=output_cross_attention_dim,
            clip_embeddings_dim=512,
            clip_extra_context_tokens=16,
        ).to(dtype=dtype)

        return (
            {
                'ipadapter': idadapter,
                'instantid': instantid
            }, 
        )

class AdapterStyleLoader:

    @classmethod
    def INPUT_TYPES(s):
        return {"required": {
        }}

    RETURN_TYPES = ("ADAPTERSTYLELOADER",)
    RETURN_NAMES = ("adapterstyle_loader", )

    FUNCTION = "load_model"
    CATEGORY = "Alkaid/Loader"

    def load_model(self):
        device = comfy.model_management.get_torch_device()
        dtype = torch.float16 if comfy.model_management.should_use_fp16() else torch.bfloat16 if comfy.model_management.should_use_bf16() else torch.float32

        ckpt_path = os.path.join(ALKAID_DIR, "style_xl.safetensors")
        model = comfy.utils.load_torch_file(ckpt_path, safe_load=True)
        if ckpt_path.lower().endswith(".safetensors"):
            st_model = {"image_proj": {}, "ip_adapter": {}}
            for key in model.keys():
                if key.startswith("image_proj."):
                    st_model["image_proj"][key.replace("image_proj.", "")] = model[key]
                elif key.startswith("ip_adapter."):
                    st_model["ip_adapter"][key.replace("ip_adapter.", "")] = model[key]
            model = st_model
            del st_model


        output_cross_attention_dim = 2048

        ipa = StyleAdapter(
            model,
            cross_attention_dim=1280,
            output_cross_attention_dim=output_cross_attention_dim,
            clip_embeddings_dim=1280,
            clip_extra_context_tokens=16,
        ).to(dtype=dtype)

        return ({
            "adapter": ipa
        }, )