support JoyCaptionBetaOne model CPU offload

py/joycaption_beta_1.py

@@ -98,11 +98,13 @@ class JoyCaptionPredictor:
             self.model = LlavaForConditionalGeneration.from_pretrained(
                 checkpoint_path,
                 torch_dtype="bfloat16",
-                device_map="auto")
+                device_map="auto"
+            )
         else:
             from transformers import BitsAndBytesConfig
             qnt_config = BitsAndBytesConfig(
                 **QUANTIZATION_CONFIGS[quantization_mode],
+                llm_int8_enable_fp32_cpu_offload=True,
                 llm_int8_skip_modules=["vision_tower", "multi_modal_projector"],
                 # Transformer's Siglip implementation has bugs when quantized, so skip those.
             )
@@ -110,7 +112,8 @@ class JoyCaptionPredictor:
                 checkpoint_path,
                 torch_dtype="auto",
                 device_map="auto",
-                quantization_config=qnt_config)
+                quantization_config=qnt_config
+            )
 
         self.model.eval()
 

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui_layerstyle_advance"
 description = "The nodes detached from ComfyUI Layer Style are mainly those with complex requirements for dependency packages."
-version = "2.0.20"
+version = "2.0.21"
 license = { text = "MIT License" }
 dependencies = ["numpy", "matplotlib", "scikit_image", "scikit_learn", "opencv-contrib-python", "pymatting", "timm", "blend_modes", "transformers", "diffusers", "loguru", "colour-science", "huggingface_hub", "segment_anything", "addict", "omegaconf", "yapf", "wget", "iopath", "mediapipe", "typer_config", "fastapi", "rich", "google-generativeai", "ultralytics", "transparent-background", "accelerate", "onnxruntime", "bitsandbytes", "peft", "protobuf", "hydra-core", "blind-watermark", "qrcode", "pyzbar", "psd-tools", "wandb", "zhipuai", "openai","google-genai"]