diff --git a/README.md b/README.md index c2977ec..1dc1f7e 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Update ----- +* support te use gguf, use less memory(all need 40G RAM ) / te支持gguf,内存占用更少(共占用40G左右,压缩了24G),模型在hg或者夸克云 * Support gguf now, use less memory / 支持gguf,内存占用更少,模型在hg或者夸克云 @@ -39,10 +40,11 @@ pip install -r requirements.txt | ├──joy_image_transformer.safetensors # optional | ├── gguf/ | ├──joy_image_transformer-Q8_0.gguf # optional +| ├──JoyAI-Image-Und-merger-Q6_K.gguf # optional te | ├── vae/ | ├──Wan2.1_VAE.pth | ├── clips -| ├──JoyAI-Image-Und-merger_bf16.safetensors +| ├──JoyAI-Image-Und-merger_bf16.safetensors # te optional ``` 4.Example @@ -51,8 +53,10 @@ pip install -r requirements.txt ![](https://github.com/smthemex/ComfyUI_JoyAI_Image/blob/main/example_workflows/example.png) ![](https://github.com/smthemex/ComfyUI_JoyAI_Image/blob/main/example_workflows/example2.png) ![](https://github.com/smthemex/ComfyUI_JoyAI_Image/blob/main/example_workflows/example3.png) -* GGUF +* dit gguf ![](https://github.com/smthemex/ComfyUI_JoyAI_Image/blob/main/example_workflows/example_q.png) +* TE gguf +![](https://github.com/smthemex/ComfyUI_JoyAI_Image/blob/main/example_workflows/example_te.png) 5.Citation ---- diff --git a/example_workflows/example_te.png b/example_workflows/example_te.png new file mode 100644 index 0000000..cf1ba8d Binary files /dev/null and b/example_workflows/example_te.png differ diff --git a/inference_und.py b/inference_und.py index 1a59a2b..deaaa2e 100644 --- a/inference_und.py +++ b/inference_und.py @@ -263,7 +263,7 @@ def load_qwen3vl_model(safetensors_path,gguf_path ) -> torch.nn.Module: model.eval().to(dtype) elif gguf_path is not None: - g_dict=load_gguf_checkpoint(gguf_path) + g_dict=load_gguf_checkpoint(gguf_path,True) match_state_dict(model, g_dict,show_num=20) set_gguf2meta_model(model,g_dict,dtype,torch.device("cpu")) del g_dict @@ -376,7 +376,7 @@ def get_conditioning(clip,prompt, images,infer_device): negative=[[n_prompt_embeds,{"prompt_attention_mask": n_prompt_embeds_mask}]] return positive,negative -def load_gguf_checkpoint(gguf_checkpoint_path): +def load_gguf_checkpoint(gguf_checkpoint_path,qwen_mode=False): import logging logging.basicConfig(level=logging.INFO) @@ -414,8 +414,11 @@ def load_gguf_checkpoint(gguf_checkpoint_path): ) weights = torch.from_numpy(tensor.data) #tensor.data.copy() - - parsed_parameters[name.replace("model.", "")] = GGUFParameter(weights, quant_type=quant_type) if is_gguf_quant else weights + if qwen_mode: + parsed_parameters[name] = GGUFParameter(weights, quant_type=quant_type) if is_gguf_quant else weights + else: + + parsed_parameters[name.replace("model.", "")] = GGUFParameter(weights, quant_type=quant_type) if is_gguf_quant else weights del tensor,weights if i > 0 and i % 1000 == 0: # 每1000个tensor执行一次gc logger.info(f"Processed {i}tensors...") diff --git a/src/modules/__init__.py b/src/modules/__init__.py deleted file mode 100644 index e69de29..0000000