kijai-ComfyUI-APISR-KJ/nodes.py

import folder_paths
import os
import torch
import torch.nn.functional as F
from .architecture.rrdb import RRDBNet
from .architecture.grl import GRL
import comfy.model_management as mm
import comfy.utils
from contextlib import nullcontext

def convert_dtype(dtype_str):
    if dtype_str == 'fp32':
        return torch.float32
    elif dtype_str == 'fp16':
        return torch.float16
    elif dtype_str == 'bf16':
        return torch.bfloat16
    else:
        raise NotImplementedError
def load_rrdb(generator_weight_PATH, scale, print_options=False):  
    ''' A simpler API to load RRDB model from Real-ESRGAN
    Args:
        generator_weight_PATH (str): The path to the weight
        scale (int): the scaling factor
        print_options (bool): whether to print options to show what kinds of setting is used
    Returns:
        generator (torch): the generator instance of the model
    '''  

    # Load the checkpoint
    checkpoint_g = torch.load(generator_weight_PATH)

    # Find the generator weight
    if 'params_ema' in checkpoint_g:
        # For official ESRNET/ESRGAN weight
        weight = checkpoint_g['params_ema']
        generator = RRDBNet(3, 3, scale=scale)    # Default blocks num is 6     

    elif 'params' in checkpoint_g:
        # For official ESRNET/ESRGAN weight
        weight = checkpoint_g['params']
        generator = RRDBNet(3, 3, scale=scale)          

    elif 'model_state_dict' in checkpoint_g:
        # For my personal trained weight
        weight = checkpoint_g['model_state_dict']
        generator = RRDBNet(3, 3, scale=scale)          

    else:
        print("This weight is not supported")
        os._exit(0)


    # Handle torch.compile weight key rename
    old_keys = [key for key in weight]
    for old_key in old_keys:
        if old_key[:10] == "_orig_mod.":
            new_key = old_key[10:]
            weight[new_key] = weight[old_key]
            del weight[old_key]

    generator.load_state_dict(weight)
    generator = generator.eval()


    # Print options to show what kinds of setting is used
    if print_options:
        if 'opt' in checkpoint_g:
            for key in checkpoint_g['opt']:
                value = checkpoint_g['opt'][key]
                print(f'{key} : {value}')

    return generator

def load_grl(generator_weight_PATH, scale=4):
    ''' A simpler API to load GRL model
    Args:
        generator_weight_PATH (str): The path to the weight
        scale (int):        Scale Factor (Usually Set as 4)
    Returns:
        generator (torch): the generator instance of the model
    '''

    # Load the checkpoint
    checkpoint_g = torch.load(generator_weight_PATH)

     # Find the generator weight
    if 'model_state_dict' in checkpoint_g:
        weight = checkpoint_g['model_state_dict']

        # GRL tiny model (Note: tiny2 version)
        generator = GRL(
            upscale = scale,
            img_size = 64,
            window_size = 8,
            depths = [4, 4, 4, 4],
            embed_dim = 64,
            num_heads_window = [2, 2, 2, 2],
            num_heads_stripe = [2, 2, 2, 2],
            mlp_ratio = 2,
            qkv_proj_type = "linear",
            anchor_proj_type = "avgpool",
            anchor_window_down_factor = 2,
            out_proj_type = "linear",
            conv_type = "1conv",
            upsampler = "nearest+conv",     # Change
        )

    else:
        print("This weight is not supported")
        os._exit(0)


    generator.load_state_dict(weight)
    generator = generator.eval()


    num_params = 0
    for p in generator.parameters():
        if p.requires_grad:
            num_params += p.numel()
    print(f"Number of parameters {num_params / 10 ** 6: 0.2f}")


    return generator

class APISR_upscale:
    @classmethod
    def INPUT_TYPES(s):
        return {"required": {
            "ckpt_name": (folder_paths.get_filename_list("upscale_models"), ),
            "images": ("IMAGE",),
            "per_batch": ("INT", {"default": 16, "min": 1, "max": 4096, "step": 1}),
            "dtype": (
                    [
                        'fp32',
                        'fp16',
                    ], {
                        "default": 'fp32'
                    }),
            },
        }

    RETURN_TYPES = ("IMAGE", )
    RETURN_NAMES = ("images", )
    FUNCTION = "upscale"
    CATEGORY = "APISR-KJ"

    def upscale(self, ckpt_name, dtype, images, per_batch):
        device = mm.get_torch_device()
        offload_device = mm.unet_offload_device()
        model_path = folder_paths.get_full_path("upscale_models", ckpt_name)
        custom_config = {
            'dtype': dtype,
            'ckpt_name': ckpt_name,
        }
        
        dtype = (convert_dtype(dtype))
        if not hasattr(self, 'model') or self.model == None or custom_config != self.current_config:
            self.model = None
            self.current_config = custom_config
            if "RRDB" in ckpt_name:
                self.model = load_rrdb(model_path, scale=2)
            elif "GRL" in ckpt_name:
                self.model = load_grl(model_path, scale=4)
            self.model = self.model.to(dtype).to(device)
   
        images = images.permute(0, 3, 1, 2)
        B, C, H, W = images.shape
        H = (H // 8) * 8
        W = (W // 8) * 8

        if images.shape[2] != H or images.shape[3] != W:
            images = F.interpolate(images, size=(H, W), mode="bilinear")
        images = images.to(device = device, dtype = dtype)
        self.model.to(device)
        pbar = comfy.utils.ProgressBar(B)
        t = []
        autocast_condition = not comfy.model_management.is_device_mps(device)
        with torch.autocast(comfy.model_management.get_autocast_device(device), dtype=dtype) if autocast_condition else nullcontext():
            for start_idx in range(0, B, per_batch):
                sub_images = self.model(images[start_idx:start_idx+per_batch])
                t.append(sub_images.cpu())
                # Calculate the number of images processed in this batch
                batch_count = sub_images.shape[0]
                # Update the progress bar by the number of images processed in this batch
                pbar.update(batch_count)
        self.model.to(offload_device)
        
        t = torch.cat(t, dim=0).permute(0, 2, 3, 1).cpu().to(torch.float32)
        
        return (t,)
        

NODE_CLASS_MAPPINGS = {
    "APISR_upscale": APISR_upscale,
}
NODE_DISPLAY_NAME_MAPPINGS = {
    "APISR_upscale": "APISR Upscale",
}