nodes/smthemex-ComfyUI_GRAG_Image_Editing
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1 from smthemex/Pr

Browse Source 
init
This commit is contained in:
smthemex
2025-11-29 18:43:25 +08:00
committed by GitHub
parent 7972589c03 0271c54f26
commit fec6bcf9de
40 changed files with 1233740 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Qwen_Edit_GRAG/Qwen-Image-Edit/model_index.json
+28
View File
@@ -0,0 +1,28 @@
{
"_class_name": "QwenImageEditPipeline",
"_diffusers_version": "0.35.0.dev0",
"processor": [
"transformers",
"Qwen2VLProcessor"
],
"scheduler": [
"diffusers",
"FlowMatchEulerDiscreteScheduler"
],
"text_encoder": [
"transformers",
"Qwen2_5_VLForConditionalGeneration"
],
"tokenizer": [
"transformers",
"Qwen2Tokenizer"
],
"transformer": [
"diffusers",
"QwenImageTransformer2DModel"
],
"vae": [
"diffusers",
"AutoencoderKLQwenImage"
]
}
Qwen_Edit_GRAG/Qwen-Image-Edit/processor/added_tokens.json
+24
View File
@@ -0,0 +1,24 @@
{
"</tool_call>": 151658,
"<tool_call>": 151657,
"<|box_end|>": 151649,
"<|box_start|>": 151648,
"<|endoftext|>": 151643,
"<|file_sep|>": 151664,
"<|fim_middle|>": 151660,
"<|fim_pad|>": 151662,
"<|fim_prefix|>": 151659,
"<|fim_suffix|>": 151661,
"<|im_end|>": 151645,
"<|im_start|>": 151644,
"<|image_pad|>": 151655,
"<|object_ref_end|>": 151647,
"<|object_ref_start|>": 151646,
"<|quad_end|>": 151651,
"<|quad_start|>": 151650,
"<|repo_name|>": 151663,
"<|video_pad|>": 151656,
"<|vision_end|>": 151653,
"<|vision_pad|>": 151654,
"<|vision_start|>": 151652
}
Qwen_Edit_GRAG/Qwen-Image-Edit/processor/chat_template.jinja
+7
View File
@@ -0,0 +1,7 @@
{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
You are a helpful assistant.<|im_end|>
{% endif %}<|im_start|>{{ message['role'] }}
{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
{% endif %}
Qwen_Edit_GRAG/Qwen-Image-Edit/processor/merges.txt
+151388
View File
File diff suppressed because it is too large Load Diff
Qwen_Edit_GRAG/Qwen-Image-Edit/processor/preprocessor_config.json
+37
View File
@@ -0,0 +1,37 @@
{
"crop_size": null,
"data_format": "channels_first",
"default_to_square": true,
"device": null,
"disable_grouping": null,
"do_center_crop": null,
"do_convert_rgb": true,
"do_normalize": true,
"do_rescale": true,
"do_resize": true,
"image_mean": [
0.48145466,
0.4578275,
0.40821073
],
"image_processor_type": "Qwen2VLImageProcessorFast",
"image_std": [
0.26862954,
0.26130258,
0.27577711
],
"input_data_format": null,
"max_pixels": 12845056,
"merge_size": 2,
"min_pixels": 3136,
"patch_size": 14,
"processor_class": "Qwen2VLProcessor",
"resample": 3,
"rescale_factor": 0.00392156862745098,
"return_tensors": null,
"size": {
"longest_edge": 12845056,
"shortest_edge": 3136
},
"temporal_patch_size": 2
}
Qwen_Edit_GRAG/Qwen-Image-Edit/processor/special_tokens_map.json
+31
View File
@@ -0,0 +1,31 @@
{
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"eos_token": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}
Qwen_Edit_GRAG/Qwen-Image-Edit/processor/tokenizer.json
+757444
View File
File diff suppressed because it is too large Load Diff
Qwen_Edit_GRAG/Qwen-Image-Edit/processor/tokenizer_config.json
+208
View File
@@ -0,0 +1,208 @@
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"151643": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151644": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151645": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151646": {
"content": "<|object_ref_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151647": {
"content": "<|object_ref_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151648": {
"content": "<|box_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151649": {
"content": "<|box_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151650": {
"content": "<|quad_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151651": {
"content": "<|quad_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151652": {
"content": "<|vision_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151653": {
"content": "<|vision_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151654": {
"content": "<|vision_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151655": {
"content": "<|image_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151656": {
"content": "<|video_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151657": {
"content": "<tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151658": {
"content": "</tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151659": {
"content": "<|fim_prefix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151660": {
"content": "<|fim_middle|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151661": {
"content": "<|fim_suffix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151662": {
"content": "<|fim_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151663": {
"content": "<|repo_name|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151664": {
"content": "<|file_sep|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
}
},
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": {},
"model_max_length": 131072,
"pad_token": "<|endoftext|>",
"processor_class": "Qwen2VLProcessor",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
}
Qwen_Edit_GRAG/Qwen-Image-Edit/processor/video_preprocessor_config.json
+43
View File
@@ -0,0 +1,43 @@
{
"crop_size": null,
"data_format": "channels_first",
"default_to_square": true,
"device": null,
"do_center_crop": null,
"do_convert_rgb": true,
"do_normalize": true,
"do_pad": null,
"do_rescale": true,
"do_resize": true,
"do_sample_frames": false,
"fps": null,
"image_mean": [
0.48145466,
0.4578275,
0.40821073
],
"image_std": [
0.26862954,
0.26130258,
0.27577711
],
"input_data_format": null,
"max_frames": 768,
"max_pixels": 12845056,
"merge_size": 2,
"min_frames": 4,
"min_pixels": 3136,
"num_frames": null,
"patch_size": 14,
"processor_class": "Qwen2VLProcessor",
"resample": 3,
"rescale_factor": 0.00392156862745098,
"size": {
"longest_edge": 12845056,
"shortest_edge": 3136
},
"size_divisor": null,
"temporal_patch_size": 2,
"video_metadata": null,
"video_processor_type": "Qwen2VLVideoProcessor"
}
Qwen_Edit_GRAG/Qwen-Image-Edit/processor/vocab.json
+1
View File
File diff suppressed because one or more lines are too long
Qwen_Edit_GRAG/Qwen-Image-Edit/scheduler/scheduler_config.json
+18
View File
@@ -0,0 +1,18 @@
{
"_class_name": "FlowMatchEulerDiscreteScheduler",
"_diffusers_version": "0.35.0.dev0",
"base_image_seq_len": 256,
"base_shift": 0.5,
"invert_sigmas": false,
"max_image_seq_len": 8192,
"max_shift": 0.9,
"num_train_timesteps": 1000,
"shift": 1.0,
"shift_terminal": 0.02,
"stochastic_sampling": false,
"time_shift_type": "exponential",
"use_beta_sigmas": false,
"use_dynamic_shifting": true,
"use_exponential_sigmas": false,
"use_karras_sigmas": false
}
Qwen_Edit_GRAG/Qwen-Image-Edit/text_encoder/config.json
+135
View File
@@ -0,0 +1,135 @@
{
"architectures": [
"Qwen2_5_VLForConditionalGeneration"
],
"attention_dropout": 0.0,
"bos_token_id": 151643,
"eos_token_id": 151645,
"hidden_act": "silu",
"hidden_size": 3584,
"image_token_id": 151655,
"initializer_range": 0.02,
"intermediate_size": 18944,
"max_position_embeddings": 128000,
"max_window_layers": 28,
"model_type": "qwen2_5_vl",
"num_attention_heads": 28,
"num_hidden_layers": 28,
"num_key_value_heads": 4,
"rms_norm_eps": 1e-06,
"rope_scaling": {
"mrope_section": [
16,
24,
24
],
"rope_type": "default",
"type": "default"
},
"rope_theta": 1000000.0,
"sliding_window": 32768,
"text_config": {
"architectures": [
"Qwen2_5_VLForConditionalGeneration"
],
"attention_dropout": 0.0,
"bos_token_id": 151643,
"eos_token_id": 151645,
"hidden_act": "silu",
"hidden_size": 3584,
"image_token_id": null,
"initializer_range": 0.02,
"intermediate_size": 18944,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 128000,
"max_window_layers": 28,
"model_type": "qwen2_5_vl_text",
"num_attention_heads": 28,
"num_hidden_layers": 28,
"num_key_value_heads": 4,
"rms_norm_eps": 1e-06,
"rope_scaling": {
"mrope_section": [
16,
24,
24
],
"rope_type": "default",
"type": "default"
},
"rope_theta": 1000000.0,
"sliding_window": null,
"torch_dtype": "float32",
"use_cache": true,
"use_sliding_window": false,
"video_token_id": null,
"vision_end_token_id": 151653,
"vision_start_token_id": 151652,
"vision_token_id": 151654,
"vocab_size": 152064
},
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.55.2",
"use_cache": true,
"use_sliding_window": false,
"video_token_id": 151656,
"vision_config": {
"depth": 32,
"fullatt_block_indexes": [
7,
15,
23,
31
],
"hidden_act": "silu",
"hidden_size": 1280,
"in_channels": 3,
"in_chans": 3,
"initializer_range": 0.02,
"intermediate_size": 3420,
"model_type": "qwen2_5_vl",
"num_heads": 16,
"out_hidden_size": 3584,
"patch_size": 14,
"spatial_merge_size": 2,
"spatial_patch_size": 14,
"temporal_patch_size": 2,
"tokens_per_second": 2,
"torch_dtype": "float32",
"window_size": 112
},
"vision_end_token_id": 151653,
"vision_start_token_id": 151652,
"vision_token_id": 151654,
"vocab_size": 152064
}
Qwen_Edit_GRAG/Qwen-Image-Edit/text_encoder/generation_config.json
+14
View File
@@ -0,0 +1,14 @@
{
"bos_token_id": 151643,
"do_sample": true,
"eos_token_id": [
151645,
151643
],
"pad_token_id": 151643,
"repetition_penalty": 1.05,
"temperature": 0.1,
"top_k": 1,
"top_p": 0.001,
"transformers_version": "4.55.2"
}
Qwen_Edit_GRAG/Qwen-Image-Edit/text_encoder/model.safetensors.index.json
+737
View File
@@ -0,0 +1,737 @@
{
"metadata": {
"total_parameters": 8292166656,
"total_size": 16584333312
},
"weight_map": {
"lm_head.weight": "model-00004-of-00004.safetensors",
"model.embed_tokens.weight": "model-00001-of-00004.safetensors",
"model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
"model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
"model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
"model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
"model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
"model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
"model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
"model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
"model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
"model.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
"model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
"model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
"model.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
"model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
"model.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
"model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
"model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
"model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
"model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.6.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.6.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.6.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.7.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.7.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.7.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
"model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
"model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
"model.norm.weight": "model-00004-of-00004.safetensors",
"visual.blocks.0.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.0.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.0.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.0.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.1.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.1.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.1.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.1.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.10.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.10.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.10.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.10.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.10.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.11.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.11.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.11.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.11.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.11.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.11.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.11.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.11.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.12.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.12.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.12.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.12.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.12.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.12.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.12.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.13.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.13.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.13.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.13.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.13.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.13.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.13.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.14.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.14.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.14.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.14.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.14.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.14.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.14.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.14.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.15.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.15.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.15.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.15.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.15.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.15.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.15.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.15.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.16.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.16.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.16.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.16.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.16.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.16.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.17.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.17.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.17.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.17.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.17.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.17.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.17.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.18.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.18.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.18.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.18.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.18.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.19.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.19.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.19.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.19.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.19.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.19.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.2.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.2.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.2.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.2.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.20.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.20.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.20.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.20.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.20.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.20.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.20.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.21.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.21.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.21.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.21.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.21.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.21.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.22.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.22.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.22.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.22.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.22.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.22.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.22.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.23.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.23.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.23.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.23.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.23.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.23.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.23.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.24.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.24.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.24.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.24.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.24.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.24.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.24.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.24.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.25.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.25.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.25.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.25.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.25.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.25.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.26.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.26.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.26.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.26.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.26.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.26.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.27.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.27.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.27.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.27.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.27.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.27.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.27.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.27.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.27.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.27.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.27.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.28.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.28.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.28.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.28.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.28.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.28.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.28.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.28.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.28.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.28.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.28.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.29.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.29.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.29.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.29.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.29.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.29.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.29.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.29.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.29.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.29.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.29.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.3.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.3.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.3.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.30.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.30.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.30.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.30.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.30.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.30.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.30.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.30.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.30.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.30.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.30.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.30.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.31.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.31.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.31.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.31.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.31.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.31.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.31.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.31.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.31.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.31.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.31.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.31.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.4.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.4.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.4.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.5.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.5.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.5.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.5.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.5.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.6.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.6.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.6.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.7.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.7.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.7.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.8.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.8.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.8.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.8.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.8.norm2.weight": "model-00001-of-00004.safetensors",
"visual.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
"visual.blocks.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
"visual.blocks.9.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.9.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.9.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
"visual.blocks.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
"visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors",
"visual.blocks.9.norm2.weight": "model-00001-of-00004.safetensors",
"visual.merger.ln_q.weight": "model-00001-of-00004.safetensors",
"visual.merger.mlp.0.bias": "model-00001-of-00004.safetensors",
"visual.merger.mlp.0.weight": "model-00001-of-00004.safetensors",
"visual.merger.mlp.2.bias": "model-00001-of-00004.safetensors",
"visual.merger.mlp.2.weight": "model-00001-of-00004.safetensors",
"visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors"
}
}
Qwen_Edit_GRAG/Qwen-Image-Edit/tokenizer/added_tokens.json
+24
View File
@@ -0,0 +1,24 @@
{
"</tool_call>": 151658,
"<tool_call>": 151657,
"<|box_end|>": 151649,
"<|box_start|>": 151648,
"<|endoftext|>": 151643,
"<|file_sep|>": 151664,
"<|fim_middle|>": 151660,
"<|fim_pad|>": 151662,
"<|fim_prefix|>": 151659,
"<|fim_suffix|>": 151661,
"<|im_end|>": 151645,
"<|im_start|>": 151644,
"<|image_pad|>": 151655,
"<|object_ref_end|>": 151647,
"<|object_ref_start|>": 151646,
"<|quad_end|>": 151651,
"<|quad_start|>": 151650,
"<|repo_name|>": 151663,
"<|video_pad|>": 151656,
"<|vision_end|>": 151653,
"<|vision_pad|>": 151654,
"<|vision_start|>": 151652
}
Qwen_Edit_GRAG/Qwen-Image-Edit/tokenizer/chat_template.jinja
+54
View File
@@ -0,0 +1,54 @@
{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0]['role'] == 'system' %}
{{- messages[0]['content'] }}
{%- else %}
{{- 'You are a helpful assistant.' }}
{%- endif %}
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
{%- else %}
{%- if messages[0]['role'] == 'system' %}
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
{%- else %}
{{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- for message in messages %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{{- '<|im_start|>' + message.role }}
{%- if message.content %}
{{- '\n' + message.content }}
{%- endif %}
{%- for tool_call in message.tool_calls %}
{%- if tool_call.function is defined %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '\n<tool_call>\n{"name": "' }}
{{- tool_call.name }}
{{- '", "arguments": ' }}
{{- tool_call.arguments | tojson }}
{{- '}\n</tool_call>' }}
{%- endfor %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n<tool_response>\n' }}
{{- message.content }}
{{- '\n</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- endif %}
Qwen_Edit_GRAG/Qwen-Image-Edit/tokenizer/merges.txt
+151388
View File
File diff suppressed because it is too large Load Diff
Qwen_Edit_GRAG/Qwen-Image-Edit/tokenizer/special_tokens_map.json
+31
View File
@@ -0,0 +1,31 @@
{
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"eos_token": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}
Qwen_Edit_GRAG/Qwen-Image-Edit/tokenizer/tokenizer_config.json
+207
View File
@@ -0,0 +1,207 @@
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"151643": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151644": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151645": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151646": {
"content": "<|object_ref_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151647": {
"content": "<|object_ref_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151648": {
"content": "<|box_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151649": {
"content": "<|box_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151650": {
"content": "<|quad_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151651": {
"content": "<|quad_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151652": {
"content": "<|vision_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151653": {
"content": "<|vision_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151654": {
"content": "<|vision_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151655": {
"content": "<|image_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151656": {
"content": "<|video_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151657": {
"content": "<tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151658": {
"content": "</tool_call>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151659": {
"content": "<|fim_prefix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151660": {
"content": "<|fim_middle|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151661": {
"content": "<|fim_suffix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151662": {
"content": "<|fim_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151663": {
"content": "<|repo_name|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"151664": {
"content": "<|file_sep|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
}
},
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": {},
"model_max_length": 131072,
"pad_token": "<|endoftext|>",
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
}
Qwen_Edit_GRAG/Qwen-Image-Edit/tokenizer/vocab.json
+151645
View File
File diff suppressed because it is too large Load Diff
Qwen_Edit_GRAG/Qwen-Image-Edit/transformer/config.json
+17
View File
@@ -0,0 +1,17 @@
{
"_class_name": "QwenImageTransformer2DModel",
"_diffusers_version": "0.35.0.dev0",
"attention_head_dim": 128,
"axes_dims_rope": [
16,
56,
56
],
"guidance_embeds": false,
"in_channels": 64,
"joint_attention_dim": 3584,
"num_attention_heads": 24,
"num_layers": 60,
"out_channels": 16,
"patch_size": 2
}
Qwen_Edit_GRAG/Qwen-Image-Edit/transformer/diffusion_pytorch_model.safetensors.index.json
+1940
View File
File diff suppressed because it is too large Load Diff
Qwen_Edit_GRAG/Qwen-Image-Edit/vae/config.json
+56
View File
@@ -0,0 +1,56 @@
{
"_class_name": "AutoencoderKLQwenImage",
"_diffusers_version": "0.35.0.dev0",
"attn_scales": [],
"base_dim": 96,
"dim_mult": [
1,
2,
4,
4
],
"dropout": 0.0,
"latents_mean": [
-0.7571,
-0.7089,
-0.9113,
0.1075,
-0.1745,
0.9653,
-0.1517,
1.5508,
0.4134,
-0.0715,
0.5517,
-0.3632,
-0.1922,
-0.9497,
0.2503,
-0.2921
],
"latents_std": [
2.8184,
1.4541,
2.3275,
2.6558,
1.2196,
1.7708,
2.6052,
2.0743,
3.2687,
2.1526,
2.8652,
1.5579,
1.6382,
1.1253,
2.8251,
1.916
],
"num_res_blocks": 2,
"temperal_downsample": [
false,
true,
true
],
"z_dim": 16
}
Qwen_Edit_GRAG/app.py
+161
View File
@@ -0,0 +1,161 @@
"""
Minimal Gradio wrapper for the given Qwen-Image-Edit inference script.
Features:
- Loads the model once and reuses it.
- Inputs: image, edit prompt, cond_b, cond_delta, optional model path.
- Matches your original settings (size 1024, steps=24, true_cfg_scale=4.0,
fixed seed=42, and the same GRAG scale structure repeated 60 times).
Run:
pip install gradio pillow torch
# plus your project deps providing hacked_models/* and model weights
python gradio_qwen_edit_minimal.py
Then open the local URL printed by Gradio.
"""
import os
from typing import Optional
import gradio as gr
import torch
from PIL import Image
from huggingface_hub import snapshot_download
import os
# --- your project imports (as in the original script) ---
from hacked_models.scheduler import FlowMatchEulerDiscreteScheduler
from hacked_models.pipeline import QwenImageEditPipeline
from hacked_models.models import QwenImageTransformer2DModel
from hacked_models.utils import seed_everything
# -----------------------------
# Global state
# -----------------------------
_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
_DTYPE = torch.bfloat16 if _DEVICE == "cuda" else torch.float32
_PIPELINE: Optional[QwenImageEditPipeline] = None
_LOADED_MODEL_PATH: Optional[str] = None
def _load_pipeline(model_path: str) -> QwenImageEditPipeline:
"""Load (or reuse) the pipeline for the given model_path."""
global _PIPELINE, _LOADED_MODEL_PATH
if _PIPELINE is not None and _LOADED_MODEL_PATH == model_path:
return _PIPELINE
# Set seed once (matches original)
seed_everything(42)
# Load components
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
os.path.join(model_path, "scheduler"), torch_dtype=_DTYPE
)
transformer = QwenImageTransformer2DModel.from_pretrained(
os.path.join(model_path, "transformer"), torch_dtype=_DTYPE
)
pipe = QwenImageEditPipeline.from_pretrained(
model_path, torch_dtype=_DTYPE, scheduler=scheduler, transformer=transformer
)
pipe.set_progress_bar_config(disable=None)
pipe.to(_DTYPE)
pipe.to(_DEVICE)
_PIPELINE = pipe
_LOADED_MODEL_PATH = model_path
return pipe
def _build_grag_scale(cond_b: float, cond_delta: float, repeats: int = 60):
"""Replicates your original GRAG schedule structure.
Each element is: ((512, 1.0, 1.0), (4096, cond_b, cond_delta))
"""
return [((512, 1.0, 1.0), (4096, cond_b, cond_delta))] * repeats
def predict(
image: Image.Image,
edit_prompt: str,
cond_b: float,
cond_delta: float,
pipeline,
):
if image is None or not edit_prompt:
return None
# Match original preprocessing
input_image = image.convert("RGB").resize((1024, 1024))
inputs = {
"image": input_image,
"prompt": edit_prompt,
"generator": torch.manual_seed(42),
"true_cfg_scale": 4.0,
"negative_prompt": " ",
"num_inference_steps": 24,
"return_dict": False,
"grag_scale": _build_grag_scale(cond_b, cond_delta, repeats=60),
}
with torch.inference_mode():
image_batch, x0_images, saved_outputs = pipe(**inputs)
# Return the first image (same as original save behavior)
return image_batch[0]
model_dir = "Qwen-Image-Edit"
repo_id = "Qwen/Qwen-Image-Edit"
if not os.path.exists(model_dir) or not os.listdir(model_dir):
snapshot_download(repo_id=repo_id, local_dir=model_dir, local_dir_use_symlinks=False)
print(f"Model downloaded to {model_dir}")
else:
print(f"Model already exists at {model_dir}")
pipe = _load_pipeline(model_dir)
with gr.Blocks(title="Qwen Image Edit — Minimal GRAG Demo") as demo:
gr.Markdown("# Qwen Image Edit — Minimal GRAG Demo\nUpload an image, enter your edit instruction, and set GRAG params.")
with gr.Row():
in_image = gr.Image(label="Input Image", type="pil")
out_image = gr.Image(label="Edited Output", type="pil")
edit_prompt = gr.Textbox(label="Edit Instruction", placeholder="e.g., Put a pair of black-framed glasses on him.")
with gr.Row():
cond_b = gr.Slider(label="cond_b", minimum=0.8, maximum=2.0, value=1.0, step=0.01)
cond_delta = gr.Slider(label="cond_delta", minimum=0.8, maximum=2.0, value=1.0, step=0.01)
run_btn = gr.Button("Run Edit")
run_btn.click(
fn=predict,
inputs=[in_image, edit_prompt, cond_b, cond_delta, pipe],
outputs=[out_image],
api_name="run_edit",
)
gr.Markdown(
"""
**Notes**
- Uses fixed seed=42 and num_inference_steps=24 to match your script.
- Resizes the input to 1024×1024 before inference (as in your code).
- `grag_scale` is built as a list of length 60 with the same tuples.
- Automatically chooses CUDA if available; otherwise runs on CPU.
"""
)
if __name__ == "__main__":
demo.queue().launch(share=True)
Qwen_Edit_GRAG/hacked_models/attention.py
+5683
View File
File diff suppressed because it is too large Load Diff
Qwen_Edit_GRAG/hacked_models/lora_conversion_utils.py
+2232
View File
File diff suppressed because it is too large Load Diff
Qwen_Edit_GRAG/hacked_models/lora_pipeline.py
+6932
View File
File diff suppressed because it is too large Load Diff
Qwen_Edit_GRAG/hacked_models/models.py
+686
View File
@@ -0,0 +1,686 @@
# Copyright 2025 Qwen-Image Team, The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import math
from typing import Any, Dict, List, Optional, Tuple, Union
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin
from diffusers.utils import USE_PEFT_BACKEND, logging, scale_lora_layers, unscale_lora_layers
from diffusers.utils.torch_utils import maybe_allow_in_graph
from diffusers.models.attention import AttentionMixin, FeedForward
from diffusers.models.attention_dispatch import dispatch_attention_fn
from diffusers.models.attention_processor import Attention
from diffusers.models.cache_utils import CacheMixin
from diffusers.models.embeddings import TimestepEmbedding, Timesteps
from diffusers.models.modeling_outputs import Transformer2DModelOutput
from diffusers.models.modeling_utils import ModelMixin
from diffusers.models.normalization import AdaLayerNormContinuous, RMSNorm
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
def _lastdim_l2norm(x: torch.Tensor, eps: float = 1e-12):
# 返回最后一维的 L2 范数，保持维度
return
def get_timestep_embedding(
timesteps: torch.Tensor,
embedding_dim: int,
flip_sin_to_cos: bool = False,
downscale_freq_shift: float = 1,
scale: float = 1,
max_period: int = 10000,
) -> torch.Tensor:
"""
This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings.
Args
timesteps (torch.Tensor):
a 1-D Tensor of N indices, one per batch element. These may be fractional.
embedding_dim (int):
the dimension of the output.
flip_sin_to_cos (bool):
Whether the embedding order should be `cos, sin` (if True) or `sin, cos` (if False)
downscale_freq_shift (float):
Controls the delta between frequencies between dimensions
scale (float):
Scaling factor applied to the embeddings.
max_period (int):
Controls the maximum frequency of the embeddings
Returns
torch.Tensor: an [N x dim] Tensor of positional embeddings.
"""
assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"
half_dim = embedding_dim // 2
exponent = -math.log(max_period) * torch.arange(
start=0, end=half_dim, dtype=torch.float32, device=timesteps.device
)
exponent = exponent / (half_dim - downscale_freq_shift)
emb = torch.exp(exponent).to(timesteps.dtype)
emb = timesteps[:, None].float() * emb[None, :]
# scale embeddings
emb = scale * emb
# concat sine and cosine embeddings
emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1)
# flip sine and cosine embeddings
if flip_sin_to_cos:
emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)
# zero pad
if embedding_dim % 2 == 1:
emb = torch.nn.functional.pad(emb, (0, 1, 0, 0))
return emb
def apply_rotary_emb_qwen(
x: torch.Tensor,
freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]],
use_real: bool = True,
use_real_unbind_dim: int = -1,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""
Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings
to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are
reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting
tensors contain rotary embeddings and are returned as real tensors.
Args:
x (`torch.Tensor`):
Query or key tensor to apply rotary embeddings. [B, S, H, D] xk (torch.Tensor): Key tensor to apply
freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],)
Returns:
Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings.
"""
if use_real:
cos, sin = freqs_cis # [S, D]
cos = cos[None, None]
sin = sin[None, None]
cos, sin = cos.to(x.device), sin.to(x.device)
if use_real_unbind_dim == -1:
# Used for flux, cogvideox, hunyuan-dit
x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, S, H, D//2]
x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3)
elif use_real_unbind_dim == -2:
# Used for Stable Audio, OmniGen, CogView4 and Cosmos
x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, S, H, D//2]
x_rotated = torch.cat([-x_imag, x_real], dim=-1)
else:
raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.")
out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype)
return out
else:
x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2))
freqs_cis = freqs_cis.unsqueeze(1)
x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3)
return x_out.type_as(x)
class QwenTimestepProjEmbeddings(nn.Module):
def __init__(self, embedding_dim):
super().__init__()
self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000)
self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)
def forward(self, timestep, hidden_states):
timesteps_proj = self.time_proj(timestep)
timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype)) # (N, D)
conditioning = timesteps_emb
return conditioning
class QwenEmbedRope(nn.Module):
def __init__(self, theta: int, axes_dim: List[int], scale_rope=False):
super().__init__()
self.theta = theta
self.axes_dim = axes_dim
pos_index = torch.arange(4096)
neg_index = torch.arange(4096).flip(0) * -1 - 1
self.pos_freqs = torch.cat(
[
self.rope_params(pos_index, self.axes_dim[0], self.theta),
self.rope_params(pos_index, self.axes_dim[1], self.theta),
self.rope_params(pos_index, self.axes_dim[2], self.theta),
],
dim=1,
)
self.neg_freqs = torch.cat(
[
self.rope_params(neg_index, self.axes_dim[0], self.theta),
self.rope_params(neg_index, self.axes_dim[1], self.theta),
self.rope_params(neg_index, self.axes_dim[2], self.theta),
],
dim=1,
)
self.rope_cache = {}
# DO NOT USING REGISTER BUFFER HERE, IT WILL CAUSE COMPLEX NUMBERS LOSE ITS IMAGINARY PART
self.scale_rope = scale_rope
def rope_params(self, index, dim, theta=10000):
"""
Args:
index: [0, 1, 2, 3] 1D Tensor representing the position index of the token
"""
assert dim % 2 == 0
freqs = torch.outer(index, 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim)))
freqs = torch.polar(torch.ones_like(freqs), freqs)
return freqs
def forward(self, video_fhw, txt_seq_lens, device):
"""
Args: video_fhw: [frame, height, width] a list of 3 integers representing the shape of the video Args:
txt_length: [bs] a list of 1 integers representing the length of the text
"""
if self.pos_freqs.device != device:
self.pos_freqs = self.pos_freqs.to(device)
self.neg_freqs = self.neg_freqs.to(device)
if isinstance(video_fhw, list):
video_fhw = video_fhw[0]
if not isinstance(video_fhw, list):
video_fhw = [video_fhw]
vid_freqs = []
max_vid_index = 0
for idx, fhw in enumerate(video_fhw):
frame, height, width = fhw
rope_key = f"{idx}_{height}_{width}"
if not torch.compiler.is_compiling():
if rope_key not in self.rope_cache:
self.rope_cache[rope_key] = self._compute_video_freqs(frame, height, width, idx)
video_freq = self.rope_cache[rope_key]
else:
video_freq = self._compute_video_freqs(frame, height, width, idx)
video_freq = video_freq.to(device)
vid_freqs.append(video_freq)
if self.scale_rope:
max_vid_index = max(height // 2, width // 2, max_vid_index)
else:
max_vid_index = max(height, width, max_vid_index)
max_len = max(txt_seq_lens)
txt_freqs = self.pos_freqs[max_vid_index : max_vid_index + max_len, ...]
vid_freqs = torch.cat(vid_freqs, dim=0)
return vid_freqs, txt_freqs
@functools.lru_cache(maxsize=None)
def _compute_video_freqs(self, frame, height, width, idx=0):
seq_lens = frame * height * width
freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1)
freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1)
freqs_frame = freqs_pos[0][idx : idx + frame].view(frame, 1, 1, -1).expand(frame, height, width, -1)
if self.scale_rope:
freqs_height = torch.cat([freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], dim=0)
freqs_height = freqs_height.view(1, height, 1, -1).expand(frame, height, width, -1)
freqs_width = torch.cat([freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], dim=0)
freqs_width = freqs_width.view(1, 1, width, -1).expand(frame, height, width, -1)
else:
freqs_height = freqs_pos[1][:height].view(1, height, 1, -1).expand(frame, height, width, -1)
freqs_width = freqs_pos[2][:width].view(1, 1, width, -1).expand(frame, height, width, -1)
freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-1).reshape(seq_lens, -1)
return freqs.clone().contiguous()
class QwenDoubleStreamAttnProcessor2_0:
"""
Attention processor for Qwen double-stream architecture, matching DoubleStreamLayerMegatron logic. This processor
implements joint attention computation where text and image streams are processed together.
"""
_attention_backend = None
def __init__(self):
if not hasattr(F, "scaled_dot_product_attention"):
raise ImportError(
"QwenDoubleStreamAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
)
def __call__(
self,
attn: Attention,
hidden_states: torch.FloatTensor, # Image stream
encoder_hidden_states: torch.FloatTensor = None, # Text stream
encoder_hidden_states_mask: torch.FloatTensor = None,
attention_mask: Optional[torch.FloatTensor] = None,
image_rotary_emb: Optional[torch.Tensor] = None,
grag_scale = ((512,1.0,1.0),(4096,1.0,1.0)),
) -> torch.FloatTensor:
if encoder_hidden_states is None:
raise ValueError("QwenDoubleStreamAttnProcessor2_0 requires encoder_hidden_states (text stream)")
seq_txt = encoder_hidden_states.shape[1]
# Compute QKV for image stream (sample projections)
img_query = attn.to_q(hidden_states)
img_key = attn.to_k(hidden_states)
img_value = attn.to_v(hidden_states)
# Compute QKV for text stream (context projections)
txt_query = attn.add_q_proj(encoder_hidden_states)
txt_key = attn.add_k_proj(encoder_hidden_states)
txt_value = attn.add_v_proj(encoder_hidden_states)
# Reshape for multi-head attention
img_query = img_query.unflatten(-1, (attn.heads, -1))
img_key = img_key.unflatten(-1, (attn.heads, -1))
img_value = img_value.unflatten(-1, (attn.heads, -1))
txt_query = txt_query.unflatten(-1, (attn.heads, -1))
txt_key = txt_key.unflatten(-1, (attn.heads, -1))
txt_value = txt_value.unflatten(-1, (attn.heads, -1))
# Apply QK normalization
if attn.norm_q is not None:
img_query = attn.norm_q(img_query)
if attn.norm_k is not None:
img_key = attn.norm_k(img_key)
if attn.norm_added_q is not None:
txt_query = attn.norm_added_q(txt_query)
if attn.norm_added_k is not None:
txt_key = attn.norm_added_k(txt_key)
# Apply RoPE
if image_rotary_emb is not None:
img_freqs, txt_freqs = image_rotary_emb
img_query = apply_rotary_emb_qwen(img_query, img_freqs, use_real=False)
img_key = apply_rotary_emb_qwen(img_key, img_freqs, use_real=False)
txt_query = apply_rotary_emb_qwen(txt_query, txt_freqs, use_real=False)
txt_key = apply_rotary_emb_qwen(txt_key, txt_freqs, use_real=False)
if grag_scale != (0,0) :
txt_srag_scale , cond_srag_scale = grag_scale
txt_len, txt_bias_scale, txt_delta_scale = txt_srag_scale
img_len, img_bias_scale, img_delta_scale = cond_srag_scale
txt_key_mean = txt_key[:,:txt_len,:,:].mean(dim=1)
cond_key_mean = img_key[:,-1*img_len:,:,:].mean(dim=1)
txt_key[:,:txt_len,:,:] = txt_bias_scale * txt_key_mean + (txt_key[:,:txt_len,:,:] - txt_key_mean) * txt_delta_scale
img_key[:,-1*img_len:,:,:] = img_bias_scale * cond_key_mean + (img_key[:,-1*img_len:,:,:] - cond_key_mean) * img_delta_scale
# print(f"Implemented GRAG: txt_srag_scale:{txt_srag_scale}, cond_grag_scale: {cond_srag_scale}\n")
joint_query = torch.cat([txt_query, img_query], dim=1)
joint_key = torch.cat([txt_key, img_key], dim=1)
joint_value = torch.cat([txt_value, img_value], dim=1)
# print(img_query.size())
hook_out = {"query":joint_query.clone().detach(),
"key":joint_key.clone().detach(),
"value":joint_value.clone().detach()}
# Compute joint attention
joint_hidden_states = dispatch_attention_fn(
joint_query,
joint_key,
joint_value,
attn_mask=attention_mask,
dropout_p=0.0,
is_causal=False,
backend=self._attention_backend,
)
# Reshape back
joint_hidden_states = joint_hidden_states.flatten(2, 3)
joint_hidden_states = joint_hidden_states.to(joint_query.dtype)
# Split attention outputs back
txt_attn_output = joint_hidden_states[:, :seq_txt, :] # Text part
img_attn_output = joint_hidden_states[:, seq_txt:, :] # Image part
# Apply output projections
img_attn_output = attn.to_out[0](img_attn_output)
if len(attn.to_out) > 1:
img_attn_output = attn.to_out[1](img_attn_output) # dropout
txt_attn_output = attn.to_add_out(txt_attn_output)
return (img_attn_output, txt_attn_output), hook_out
@maybe_allow_in_graph
class QwenImageTransformerBlock(nn.Module):
def __init__(
self, dim: int, num_attention_heads: int, attention_head_dim: int, qk_norm: str = "rms_norm", eps: float = 1e-6
):
super().__init__()
self.dim = dim
self.num_attention_heads = num_attention_heads
self.attention_head_dim = attention_head_dim
# Image processing modules
self.img_mod = nn.Sequential(
nn.SiLU(),
nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2
)
self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
self.attn = Attention(
query_dim=dim,
cross_attention_dim=None, # Enable cross attention for joint computation
added_kv_proj_dim=dim, # Enable added KV projections for text stream
dim_head=attention_head_dim,
heads=num_attention_heads,
out_dim=dim,
context_pre_only=False,
bias=True,
processor=QwenDoubleStreamAttnProcessor2_0(),
qk_norm=qk_norm,
eps=eps,
)
self.img_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
self.img_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
# Text processing modules
self.txt_mod = nn.Sequential(
nn.SiLU(),
nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2
)
self.txt_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
# Text doesn't need separate attention - it's handled by img_attn joint computation
self.txt_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
self.txt_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
def _modulate(self, x, mod_params):
"""Apply modulation to input tensor"""
shift, scale, gate = mod_params.chunk(3, dim=-1)
return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1), gate.unsqueeze(1)
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
encoder_hidden_states_mask: torch.Tensor,
temb: torch.Tensor,
image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
joint_attention_kwargs: Optional[Dict[str, Any]] = None,
grag_scale = ((512,1.0,1.0),(512,1.0,1.0)),
) -> Tuple[torch.Tensor, torch.Tensor]:
# Get modulation parameters for both streams
img_mod_params = self.img_mod(temb) # [B, 6*dim]
txt_mod_params = self.txt_mod(temb) # [B, 6*dim]
# Split modulation parameters for norm1 and norm2
img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1) # Each [B, 3*dim]
txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim]
# Process image stream - norm1 + modulation
img_normed = self.img_norm1(hidden_states)
img_modulated, img_gate1 = self._modulate(img_normed, img_mod1)
# Process text stream - norm1 + modulation
txt_normed = self.txt_norm1(encoder_hidden_states)
txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1)
# Use QwenAttnProcessor2_0 for joint attention computation
# This directly implements the DoubleStreamLayerMegatron logic:
# 1. Computes QKV for both streams
# 2. Applies QK normalization and RoPE
# 3. Concatenates and runs joint attention
# 4. Splits results back to separate streams
joint_attention_kwargs = joint_attention_kwargs or {}
attn_output, hook_out = self.attn(
hidden_states=img_modulated, # Image stream (will be processed as "sample")
encoder_hidden_states=txt_modulated, # Text stream (will be processed as "context")
encoder_hidden_states_mask=encoder_hidden_states_mask,
image_rotary_emb=image_rotary_emb,
grag_scale = grag_scale,
**joint_attention_kwargs,
)
hook_out = None
del hook_out
# QwenAttnProcessor2_0 returns (img_output, txt_output) when encoder_hidden_states is provided
img_attn_output, txt_attn_output = attn_output
# Apply attention gates and add residual (like in Megatron)
hidden_states = hidden_states + img_gate1 * img_attn_output
encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output
# Process image stream - norm2 + MLP
img_normed2 = self.img_norm2(hidden_states)
img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2)
img_mlp_output = self.img_mlp(img_modulated2)
hidden_states = hidden_states + img_gate2 * img_mlp_output
# Process text stream - norm2 + MLP
txt_normed2 = self.txt_norm2(encoder_hidden_states)
txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2)
txt_mlp_output = self.txt_mlp(txt_modulated2)
encoder_hidden_states = encoder_hidden_states + txt_gate2 * txt_mlp_output
# Clip to prevent overflow for fp16
if encoder_hidden_states.dtype == torch.float16:
encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504)
if hidden_states.dtype == torch.float16:
hidden_states = hidden_states.clip(-65504, 65504)
return encoder_hidden_states, hidden_states
class QwenImageTransformer2DModel(
ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin, CacheMixin, AttentionMixin
):
"""
The Transformer model introduced in Qwen.
Args:
patch_size (`int`, defaults to `2`):
Patch size to turn the input data into small patches.
in_channels (`int`, defaults to `64`):
The number of channels in the input.
out_channels (`int`, *optional*, defaults to `None`):
The number of channels in the output. If not specified, it defaults to `in_channels`.
num_layers (`int`, defaults to `60`):
The number of layers of dual stream DiT blocks to use.
attention_head_dim (`int`, defaults to `128`):
The number of dimensions to use for each attention head.
num_attention_heads (`int`, defaults to `24`):
The number of attention heads to use.
joint_attention_dim (`int`, defaults to `3584`):
The number of dimensions to use for the joint attention (embedding/channel dimension of
`encoder_hidden_states`).
guidance_embeds (`bool`, defaults to `False`):
Whether to use guidance embeddings for guidance-distilled variant of the model.
axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`):
The dimensions to use for the rotary positional embeddings.
"""
_supports_gradient_checkpointing = True
_no_split_modules = ["QwenImageTransformerBlock"]
_skip_layerwise_casting_patterns = ["pos_embed", "norm"]
_repeated_blocks = ["QwenImageTransformerBlock"]
@register_to_config
def __init__(
self,
patch_size: int = 2,
in_channels: int = 64,
out_channels: Optional[int] = 16,
num_layers: int = 60,
attention_head_dim: int = 128,
num_attention_heads: int = 24,
joint_attention_dim: int = 3584,
guidance_embeds: bool = False, # TODO: this should probably be removed
axes_dims_rope: Tuple[int, int, int] = (16, 56, 56),
):
super().__init__()
self.out_channels = out_channels or in_channels
self.inner_dim = num_attention_heads * attention_head_dim
self.pos_embed = QwenEmbedRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True)
self.time_text_embed = QwenTimestepProjEmbeddings(embedding_dim=self.inner_dim)
self.txt_norm = RMSNorm(joint_attention_dim, eps=1e-6)
self.img_in = nn.Linear(in_channels, self.inner_dim)
self.txt_in = nn.Linear(joint_attention_dim, self.inner_dim)
self.transformer_blocks = nn.ModuleList(
[
QwenImageTransformerBlock(
dim=self.inner_dim,
num_attention_heads=num_attention_heads,
attention_head_dim=attention_head_dim,
)
for _ in range(num_layers)
]
)
self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6)
self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True)
self.gradient_checkpointing = False
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor = None,
encoder_hidden_states_mask: torch.Tensor = None,
timestep: torch.LongTensor = None,
img_shapes: Optional[List[Tuple[int, int, int]]] = None,
txt_seq_lens: Optional[List[int]] = None,
guidance: torch.Tensor = None, # TODO: this should probably be removed
attention_kwargs: Optional[Dict[str, Any]] = None,
controlnet_block_samples=None,
return_dict: bool = True,
grag_scale = [((512,1.0,1.0),(512,1.0,1.0))] * 60,
) -> Union[torch.Tensor, Transformer2DModelOutput]:
"""
The [`QwenTransformer2DModel`] forward method.
Args:
hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`):
Input `hidden_states`.
encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`):
Conditional embeddings (embeddings computed from the input conditions such as prompts) to use.
encoder_hidden_states_mask (`torch.Tensor` of shape `(batch_size, text_sequence_length)`):
Mask of the input conditions.
timestep ( `torch.LongTensor`):
Used to indicate denoising step.
attention_kwargs (`dict`, *optional*):
A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
`self.processor` in
[diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
return_dict (`bool`, *optional*, defaults to `True`):
Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain
tuple.
Returns:
If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a
`tuple` where the first element is the sample tensor.
"""
if attention_kwargs is not None:
attention_kwargs = attention_kwargs.copy()
lora_scale = attention_kwargs.pop("scale", 1.0)
else:
lora_scale = 1.0
if USE_PEFT_BACKEND:
# weight the lora layers by setting `lora_scale` for each PEFT layer
scale_lora_layers(self, lora_scale)
else:
if attention_kwargs is not None and attention_kwargs.get("scale", None) is not None:
logger.warning(
"Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective."
)
hidden_states = self.img_in(hidden_states)
timestep = timestep.to(hidden_states.dtype)
encoder_hidden_states = self.txt_norm(encoder_hidden_states)
encoder_hidden_states = self.txt_in(encoder_hidden_states)
if guidance is not None:
guidance = guidance.to(hidden_states.dtype) * 1000
temb = (
self.time_text_embed(timestep, hidden_states)
if guidance is None
else self.time_text_embed(timestep, guidance, hidden_states)
)
image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
for index_block, block in enumerate(self.transformer_blocks):
if torch.is_grad_enabled() and self.gradient_checkpointing:
encoder_hidden_states, hidden_states = self._gradient_checkpointing_func(
block,
hidden_states,
encoder_hidden_states,
encoder_hidden_states_mask,
temb,
image_rotary_emb,
)
else:
encoder_hidden_states, hidden_states = block(
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
encoder_hidden_states_mask=encoder_hidden_states_mask,
temb=temb,
image_rotary_emb=image_rotary_emb,
joint_attention_kwargs=attention_kwargs,
grag_scale = grag_scale.pop() ,
)
# controlnet residual
if controlnet_block_samples is not None:
interval_control = len(self.transformer_blocks) / len(controlnet_block_samples)
interval_control = int(np.ceil(interval_control))
hidden_states = hidden_states + controlnet_block_samples[index_block // interval_control]
# Use only the image part (hidden_states) from the dual-stream blocks
hidden_states = self.norm_out(hidden_states, temb)
output = self.proj_out(hidden_states)
if USE_PEFT_BACKEND:
# remove `lora_scale` from each PEFT layer
unscale_lora_layers(self, lora_scale)
if not return_dict:
return (output,)
return Transformer2DModelOutput(sample=output)
Qwen_Edit_GRAG/hacked_models/pipeline.py
+1037
View File
File diff suppressed because it is too large Load Diff
Qwen_Edit_GRAG/hacked_models/scheduler.py
+564
View File
@@ -0,0 +1,564 @@
# Copyright 2025 Stability AI, Katherine Crowson and The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union
import numpy as np
import torch
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.utils import BaseOutput, is_scipy_available, logging
from diffusers.schedulers.scheduling_utils import SchedulerMixin
if is_scipy_available():
import scipy.stats
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
@dataclass
class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput):
"""
Output class for the scheduler's `step` function output.
Args:
prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the
denoising loop.
"""
prev_sample: torch.FloatTensor
class FlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin):
"""
Euler scheduler.
This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic
methods the library implements for all schedulers such as loading and saving.
Args:
num_train_timesteps (`int`, defaults to 1000):
The number of diffusion steps to train the model.
shift (`float`, defaults to 1.0):
The shift value for the timestep schedule.
use_dynamic_shifting (`bool`, defaults to False):
Whether to apply timestep shifting on-the-fly based on the image resolution.
base_shift (`float`, defaults to 0.5):
Value to stabilize image generation. Increasing `base_shift` reduces variation and image is more consistent
with desired output.
max_shift (`float`, defaults to 1.15):
Value change allowed to latent vectors. Increasing `max_shift` encourages more variation and image may be
more exaggerated or stylized.
base_image_seq_len (`int`, defaults to 256):
The base image sequence length.
max_image_seq_len (`int`, defaults to 4096):
The maximum image sequence length.
invert_sigmas (`bool`, defaults to False):
Whether to invert the sigmas.
shift_terminal (`float`, defaults to None):
The end value of the shifted timestep schedule.
use_karras_sigmas (`bool`, defaults to False):
Whether to use Karras sigmas for step sizes in the noise schedule during sampling.
use_exponential_sigmas (`bool`, defaults to False):
Whether to use exponential sigmas for step sizes in the noise schedule during sampling.
use_beta_sigmas (`bool`, defaults to False):
Whether to use beta sigmas for step sizes in the noise schedule during sampling.
time_shift_type (`str`, defaults to "exponential"):
The type of dynamic resolution-dependent timestep shifting to apply. Either "exponential" or "linear".
stochastic_sampling (`bool`, defaults to False):
Whether to use stochastic sampling.
"""
_compatibles = []
order = 1
@register_to_config
def __init__(
self,
num_train_timesteps: int = 1000,
shift: float = 1.0,
use_dynamic_shifting: bool = False,
base_shift: Optional[float] = 0.5,
max_shift: Optional[float] = 1.15,
base_image_seq_len: Optional[int] = 256,
max_image_seq_len: Optional[int] = 4096,
invert_sigmas: bool = False,
shift_terminal: Optional[float] = None,
use_karras_sigmas: Optional[bool] = False,
use_exponential_sigmas: Optional[bool] = False,
use_beta_sigmas: Optional[bool] = False,
time_shift_type: str = "exponential",
stochastic_sampling: bool = False,
):
if self.config.use_beta_sigmas and not is_scipy_available():
raise ImportError("Make sure to install scipy if you want to use beta sigmas.")
if sum([self.config.use_beta_sigmas, self.config.use_exponential_sigmas, self.config.use_karras_sigmas]) > 1:
raise ValueError(
"Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, `config.use_karras_sigmas` can be used."
)
if time_shift_type not in {"exponential", "linear"}:
raise ValueError("`time_shift_type` must either be 'exponential' or 'linear'.")
timesteps = np.linspace(1, num_train_timesteps, num_train_timesteps, dtype=np.float32)[::-1].copy()
timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32)
sigmas = timesteps / num_train_timesteps
if not use_dynamic_shifting:
# when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution
sigmas = shift * sigmas / (1 + (shift - 1) * sigmas)
self.timesteps = sigmas * num_train_timesteps
self._step_index = None
self._begin_index = None
self._shift = shift
self.sigmas = sigmas.to("cpu") # to avoid too much CPU/GPU communication
self.sigma_min = self.sigmas[-1].item()
self.sigma_max = self.sigmas[0].item()
@property
def shift(self):
"""
The value used for shifting.
"""
return self._shift
@property
def step_index(self):
"""
The index counter for current timestep. It will increase 1 after each scheduler step.
"""
return self._step_index
@property
def begin_index(self):
"""
The index for the first timestep. It should be set from pipeline with `set_begin_index` method.
"""
return self._begin_index
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index
def set_begin_index(self, begin_index: int = 0):
"""
Sets the begin index for the scheduler. This function should be run from pipeline before the inference.
Args:
begin_index (`int`):
The begin index for the scheduler.
"""
self._begin_index = begin_index
def set_shift(self, shift: float):
self._shift = shift
def scale_noise(
self,
sample: torch.FloatTensor,
timestep: Union[float, torch.FloatTensor],
noise: Optional[torch.FloatTensor] = None,
) -> torch.FloatTensor:
"""
Forward process in flow-matching
Args:
sample (`torch.FloatTensor`):
The input sample.
timestep (`int`, *optional*):
The current timestep in the diffusion chain.
Returns:
`torch.FloatTensor`:
A scaled input sample.
"""
# Make sure sigmas and timesteps have the same device and dtype as original_samples
sigmas = self.sigmas.to(device=sample.device, dtype=sample.dtype)
if sample.device.type == "mps" and torch.is_floating_point(timestep):
# mps does not support float64
schedule_timesteps = self.timesteps.to(sample.device, dtype=torch.float32)
timestep = timestep.to(sample.device, dtype=torch.float32)
else:
schedule_timesteps = self.timesteps.to(sample.device)
timestep = timestep.to(sample.device)
# self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
if self.begin_index is None:
step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timestep]
elif self.step_index is not None:
# add_noise is called after first denoising step (for inpainting)
step_indices = [self.step_index] * timestep.shape[0]
else:
# add noise is called before first denoising step to create initial latent(img2img)
step_indices = [self.begin_index] * timestep.shape[0]
sigma = sigmas[step_indices].flatten()
while len(sigma.shape) < len(sample.shape):
sigma = sigma.unsqueeze(-1)
sample = sigma * noise + (1.0 - sigma) * sample
return sample
def _sigma_to_t(self, sigma):
return sigma * self.config.num_train_timesteps
def time_shift(self, mu: float, sigma: float, t: torch.Tensor):
if self.config.time_shift_type == "exponential":
return self._time_shift_exponential(mu, sigma, t)
elif self.config.time_shift_type == "linear":
return self._time_shift_linear(mu, sigma, t)
def stretch_shift_to_terminal(self, t: torch.Tensor) -> torch.Tensor:
r"""
Stretches and shifts the timestep schedule to ensure it terminates at the configured `shift_terminal` config
value.
Reference:
https://github.com/Lightricks/LTX-Video/blob/a01a171f8fe3d99dce2728d60a73fecf4d4238ae/ltx_video/schedulers/rf.py#L51
Args:
t (`torch.Tensor`):
A tensor of timesteps to be stretched and shifted.
Returns:
`torch.Tensor`:
A tensor of adjusted timesteps such that the final value equals `self.config.shift_terminal`.
"""
one_minus_z = 1 - t
scale_factor = one_minus_z[-1] / (1 - self.config.shift_terminal)
stretched_t = 1 - (one_minus_z / scale_factor)
return stretched_t
def set_timesteps(
self,
num_inference_steps: Optional[int] = None,
device: Union[str, torch.device] = None,
sigmas: Optional[List[float]] = None,
mu: Optional[float] = None,
timesteps: Optional[List[float]] = None,
):
"""
Sets the discrete timesteps used for the diffusion chain (to be run before inference).
Args:
num_inference_steps (`int`, *optional*):
The number of diffusion steps used when generating samples with a pre-trained model.
device (`str` or `torch.device`, *optional*):
The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
sigmas (`List[float]`, *optional*):
Custom values for sigmas to be used for each diffusion step. If `None`, the sigmas are computed
automatically.
mu (`float`, *optional*):
Determines the amount of shifting applied to sigmas when performing resolution-dependent timestep
shifting.
timesteps (`List[float]`, *optional*):
Custom values for timesteps to be used for each diffusion step. If `None`, the timesteps are computed
automatically.
"""
if self.config.use_dynamic_shifting and mu is None:
raise ValueError("`mu` must be passed when `use_dynamic_shifting` is set to be `True`")
if sigmas is not None and timesteps is not None:
if len(sigmas) != len(timesteps):
raise ValueError("`sigmas` and `timesteps` should have the same length")
if num_inference_steps is not None:
if (sigmas is not None and len(sigmas) != num_inference_steps) or (
timesteps is not None and len(timesteps) != num_inference_steps
):
raise ValueError(
"`sigmas` and `timesteps` should have the same length as num_inference_steps, if `num_inference_steps` is provided"
)
else:
num_inference_steps = len(sigmas) if sigmas is not None else len(timesteps)
self.num_inference_steps = num_inference_steps
# 1. Prepare default sigmas
is_timesteps_provided = timesteps is not None
if is_timesteps_provided:
timesteps = np.array(timesteps).astype(np.float32)
if sigmas is None:
if timesteps is None:
timesteps = np.linspace(
self._sigma_to_t(self.sigma_max), self._sigma_to_t(self.sigma_min), num_inference_steps
)
sigmas = timesteps / self.config.num_train_timesteps
else:
sigmas = np.array(sigmas).astype(np.float32)
num_inference_steps = len(sigmas)
# 2. Perform timestep shifting. Either no shifting is applied, or resolution-dependent shifting of
# "exponential" or "linear" type is applied
if self.config.use_dynamic_shifting:
sigmas = self.time_shift(mu, 1.0, sigmas)
else:
sigmas = self.shift * sigmas / (1 + (self.shift - 1) * sigmas)
# 3. If required, stretch the sigmas schedule to terminate at the configured `shift_terminal` value
if self.config.shift_terminal:
sigmas = self.stretch_shift_to_terminal(sigmas)
# 4. If required, convert sigmas to one of karras, exponential, or beta sigma schedules
if self.config.use_karras_sigmas:
sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps)
elif self.config.use_exponential_sigmas:
sigmas = self._convert_to_exponential(in_sigmas=sigmas, num_inference_steps=num_inference_steps)
elif self.config.use_beta_sigmas:
sigmas = self._convert_to_beta(in_sigmas=sigmas, num_inference_steps=num_inference_steps)
# 5. Convert sigmas and timesteps to tensors and move to specified device
sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32, device=device)
if not is_timesteps_provided:
timesteps = sigmas * self.config.num_train_timesteps
else:
timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32, device=device)
# 6. Append the terminal sigma value.
# If a model requires inverted sigma schedule for denoising but timesteps without inversion, the
# `invert_sigmas` flag can be set to `True`. This case is only required in Mochi
if self.config.invert_sigmas:
sigmas = 1.0 - sigmas
timesteps = sigmas * self.config.num_train_timesteps
sigmas = torch.cat([sigmas, torch.ones(1, device=sigmas.device)])
else:
sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)])
self.timesteps = timesteps
self.sigmas = sigmas
self._step_index = None
self._begin_index = None
def index_for_timestep(self, timestep, schedule_timesteps=None):
if schedule_timesteps is None:
schedule_timesteps = self.timesteps
indices = (schedule_timesteps == timestep).nonzero()
# The sigma index that is taken for the **very** first `step`
# is always the second index (or the last index if there is only 1)
# This way we can ensure we don't accidentally skip a sigma in
# case we start in the middle of the denoising schedule (e.g. for image-to-image)
pos = 1 if len(indices) > 1 else 0
return indices[pos].item()
def _init_step_index(self, timestep):
if self.begin_index is None:
if isinstance(timestep, torch.Tensor):
timestep = timestep.to(self.timesteps.device)
self._step_index = self.index_for_timestep(timestep)
else:
self._step_index = self._begin_index
def step(
self,
model_output: torch.FloatTensor,
timestep: Union[float, torch.FloatTensor],
sample: torch.FloatTensor,
s_churn: float = 0.0,
s_tmin: float = 0.0,
s_tmax: float = float("inf"),
s_noise: float = 1.0,
generator: Optional[torch.Generator] = None,
per_token_timesteps: Optional[torch.Tensor] = None,
return_dict: bool = True,
) -> Union[FlowMatchEulerDiscreteSchedulerOutput, Tuple]:
"""
Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
process from the learned model outputs (most often the predicted noise).
Args:
model_output (`torch.FloatTensor`):
The direct output from learned diffusion model.
timestep (`float`):
The current discrete timestep in the diffusion chain.
sample (`torch.FloatTensor`):
A current instance of a sample created by the diffusion process.
s_churn (`float`):
s_tmin (`float`):
s_tmax (`float`):
s_noise (`float`, defaults to 1.0):
Scaling factor for noise added to the sample.
generator (`torch.Generator`, *optional*):
A random number generator.
per_token_timesteps (`torch.Tensor`, *optional*):
The timesteps for each token in the sample.
return_dict (`bool`):
Whether or not to return a
[`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] or tuple.
Returns:
[`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] or `tuple`:
If return_dict is `True`,
[`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] is returned,
otherwise a tuple is returned where the first element is the sample tensor.
"""
if (
isinstance(timestep, int)
or isinstance(timestep, torch.IntTensor)
or isinstance(timestep, torch.LongTensor)
):
raise ValueError(
(
"Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
" `FlowMatchEulerDiscreteScheduler.step()` is not supported. Make sure to pass"
" one of the `scheduler.timesteps` as a timestep."
),
)
if self.step_index is None:
self._init_step_index(timestep)
# Upcast to avoid precision issues when computing prev_sample
sample = sample.to(torch.float32)
if per_token_timesteps is not None:
per_token_sigmas = per_token_timesteps / self.config.num_train_timesteps
sigmas = self.sigmas[:, None, None]
lower_mask = sigmas < per_token_sigmas[None] - 1e-6
lower_sigmas = lower_mask * sigmas
lower_sigmas, _ = lower_sigmas.max(dim=0)
current_sigma = per_token_sigmas[..., None]
next_sigma = lower_sigmas[..., None]
dt = current_sigma - next_sigma
else:
sigma_idx = self.step_index
sigma = self.sigmas[sigma_idx]
sigma_next = self.sigmas[sigma_idx + 1]
current_sigma = sigma
next_sigma = sigma_next
dt = sigma_next - sigma
if self.config.stochastic_sampling:
x0 = sample - current_sigma * model_output
noise = torch.randn_like(sample)
prev_sample = (1.0 - next_sigma) * x0 + next_sigma * noise
else:
prev_sample = sample + dt * model_output
pred_original_sample = sample + (self.sigmas[-1] - sigma) * model_output
# upon completion increase step index by one
self._step_index += 1
if per_token_timesteps is None:
# Cast sample back to model compatible dtype
prev_sample = prev_sample.to(model_output.dtype)
pred_original_sample = pred_original_sample.to(model_output.dtype)
if not return_dict:
return (prev_sample,pred_original_sample)
return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample)
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras
def _convert_to_karras(self, in_sigmas: torch.Tensor, num_inference_steps) -> torch.Tensor:
"""Constructs the noise schedule of Karras et al. (2022)."""
# Hack to make sure that other schedulers which copy this function don't break
# TODO: Add this logic to the other schedulers
if hasattr(self.config, "sigma_min"):
sigma_min = self.config.sigma_min
else:
sigma_min = None
if hasattr(self.config, "sigma_max"):
sigma_max = self.config.sigma_max
else:
sigma_max = None
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
rho = 7.0 # 7.0 is the value used in the paper
ramp = np.linspace(0, 1, num_inference_steps)
min_inv_rho = sigma_min ** (1 / rho)
max_inv_rho = sigma_max ** (1 / rho)
sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
return sigmas
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential
def _convert_to_exponential(self, in_sigmas: torch.Tensor, num_inference_steps: int) -> torch.Tensor:
"""Constructs an exponential noise schedule."""
# Hack to make sure that other schedulers which copy this function don't break
# TODO: Add this logic to the other schedulers
if hasattr(self.config, "sigma_min"):
sigma_min = self.config.sigma_min
else:
sigma_min = None
if hasattr(self.config, "sigma_max"):
sigma_max = self.config.sigma_max
else:
sigma_max = None
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
sigmas = np.exp(np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps))
return sigmas
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta
def _convert_to_beta(
self, in_sigmas: torch.Tensor, num_inference_steps: int, alpha: float = 0.6, beta: float = 0.6
) -> torch.Tensor:
"""From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)"""
# Hack to make sure that other schedulers which copy this function don't break
# TODO: Add this logic to the other schedulers
if hasattr(self.config, "sigma_min"):
sigma_min = self.config.sigma_min
else:
sigma_min = None
if hasattr(self.config, "sigma_max"):
sigma_max = self.config.sigma_max
else:
sigma_max = None
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
sigmas = np.array(
[
sigma_min + (ppf * (sigma_max - sigma_min))
for ppf in [
scipy.stats.beta.ppf(timestep, alpha, beta)
for timestep in 1 - np.linspace(0, 1, num_inference_steps)
]
]
)
return sigmas
def _time_shift_exponential(self, mu, sigma, t):
return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma)
def _time_shift_linear(self, mu, sigma, t):
return mu / (mu + (1 / t - 1) ** sigma)
def __len__(self):
return self.config.num_train_timesteps
Qwen_Edit_GRAG/hacked_models/utils.py
+35
View File
@@ -0,0 +1,35 @@
import math
from PIL import Image, ImageDraw, ImageFont
import torch
import torch.nn.functional as F
import matplotlib.pyplot as plt
import numpy as np
from typing import Dict, List, Literal
import matplotlib.cm as cm
import matplotlib as mpl
from mpl_toolkits.mplot3d import Axes3D
import os
from pathlib import Path
from datetime import datetime
from base64 import b64encode
from PIL import Image
import random
import io
from io import BytesIO
def seed_everything(seed: int = 42, deterministic: bool = False):
random.seed(seed)
os.environ["PYTHONHASHSEED"] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
if deterministic:
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
else:
torch.backends.cudnn.deterministic = False
torch.backends.cudnn.benchmark = True
print(f"✅ Random seed set to {seed}, deterministic={deterministic}")
Qwen_Edit_GRAG/inference.py
+158
View File
@@ -0,0 +1,158 @@
import os
import torch
import sys
from termcolor import colored
from diffusers import QwenImageEditPipeline
from .hacked_models.scheduler import FlowMatchEulerDiscreteScheduler
from .hacked_models.pipeline import QwenImageEditPipeline
from .hacked_models.models import QwenImageTransformer2DModel
import sys
from .hacked_models.utils import *
from contextlib import contextmanager
import sys
@contextmanager
def temp_patch_module_attr(module_name: str, attr_name: str, new_obj):
mod = sys.modules.get(module_name)
if mod is None:
yield
return
had = hasattr(mod, attr_name)
orig = getattr(mod, attr_name, None)
setattr(mod, attr_name, new_obj)
try:
yield
finally:
if had:
setattr(mod, attr_name, orig)
else:
try:
delattr(mod, attr_name)
except Exception:
pass
def load_model(gguf_path,unet_path,node_path):
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit/scheduler"),torch_dtype=torch.bfloat16,)
if gguf_path is not None:
from diffusers import GGUFQuantizationConfig
with temp_patch_module_attr("diffusers", "QwenImageTransformer2DModel", QwenImageTransformer2DModel):
transformer = QwenImageTransformer2DModel.from_single_file(
gguf_path,
config=os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit/transformer"),
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
torch_dtype=torch.bfloat16,
)
else:
try:
transformer = QwenImageTransformer2DModel.from_single_file(gguf_path,config=os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit/transformer"),torch_dtype=torch.bfloat16,)
except:
print("loading from safetensors")
from safetensors.torch import load_file
t_state_dict=load_file(unet_path)
new_dict=replace_key(t_state_dict)
with temp_patch_module_attr("diffusers", "QwenImageTransformer2DModel", QwenImageTransformer2DModel):
unet_config = QwenImageTransformer2DModel.load_config(os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit/transformer/config.json"))
transformer = QwenImageTransformer2DModel.from_config(unet_config).to(torch.bfloat16)
transformer.load_state_dict(new_dict, strict=False)
del t_state_dict,new_dict
pipeline = QwenImageEditPipeline.from_pretrained(os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit"), scheduler = scheduler,vae=None,text_encoder=None,transformer=transformer,torch_dtype=torch.bfloat16,)
return pipeline
def replace_key(t_state_dict):
return {k.replace("model.diffusion_model.", "", 1): v for k, v in t_state_dict.items()}
def inference(pipeline,positive,negative,num_inference_steps,seed,true_cfg_scale,cond_b,cond_delta):
seed_everything(seed)
pipeline.set_progress_bar_config(disable=None)
inputs = {
"image":None,
"prompt": None,
"generator": torch.manual_seed(seed),
"true_cfg_scale": true_cfg_scale,
"negative_prompt": None,
"num_inference_steps": num_inference_steps,
"prompt_embeds": positive[0][0], #pooled_prompt_embeds=positive[0][1].get("pooled_output")
"negative_prompt_embeds": negative[0][0],
"image_latents":positive[0][1].get("ref_latents",None) ,
"return_dict": False,
"grag_scale":[((512,1.0,1.0),(4096,cond_b,cond_delta))]*60,
}
with torch.inference_mode():
output = pipeline(**inputs)
image,x0_images,saved_outputs = output
#image[0].save(os.path.join(out_path,f"{args.image_path.split('/')[-1]}_cond_b-{args.cond_b}_cond_delta-{args.cond_delta}.jpg"))
return image,x0_images
#parser = argparse.ArgumentParser()
# parser.add_argument("--model_path", type=str, default="Qwen/Qwen-Image-Edit")
# parser.add_argument("--image_path", type=str, required=True)
# parser.add_argument("--edit_prompt", type=str, required=True)
# parser.add_argument("--out_path", type=str, default='./results')
# parser.add_argument("--cond_b", type=float, required=True)
# parser.add_argument("--cond_delta", type=float, required=True)
# args = parser.parse_args()
# scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
# os.path.join(args.model_path, "scheduler"),
# torch_dtype=torch.bfloat16,
# )
# transformer = QwenImageTransformer2DModel.from_pretrained(
# os.path.join(args.model_path, "transformer"),
# torch_dtype=torch.bfloat16,
# )
# pipeline = QwenImageEditPipeline.from_pretrained(args.model_path, torch_dtype=torch.bfloat16,
# scheduler = scheduler,
# transformer=transformer,
# )
# print("pipeline loaded")
# pipeline.to(torch.bfloat16)
# pipeline.to("cuda")
# pipeline.set_progress_bar_config(disable=None)
# out_path = args.out_path
# os.makedirs(out_path,exist_ok=True)
# print(colored(out_path,color = "green"))
# editing_instruction = args.edit_prompt
# input_image = Image.open(args.image_path).convert('RGB').resize((1024,1024))
# os.makedirs(os.path.join(out_path),exist_ok=True)
# prompt = editing_instruction
# inputs = {
# "image": input_image,
# "prompt": prompt,
# "generator": torch.manual_seed(42),
# "true_cfg_scale": 4.0,
# "negative_prompt": " ",
# "num_inference_steps": 24,
# "return_dict": False,
# "grag_scale":[((512,1.0,1.0),(4096,args.cond_b,args.cond_delta))]*60,
# }
# with torch.inference_mode():
# output = pipeline(**inputs)
# image,x0_images,saved_outputs = output
# image[0].save(os.path.join(out_path,f"{args.image_path.split('/')[-1]}_cond_b-{args.cond_b}_cond_delta-{args.cond_delta}.jpg"))
Qwen_Edit_GRAG/requirements.txt
+15
View File
@@ -0,0 +1,15 @@
accelerate
diffusers
gradio
ipykernel
jaraco.collections
matplotlib
pathlib
pickleshare
pip-chill
termcolor
tomli
torch
torchvision
transformers
huggingface_hub
Qwen_Edit_GRAG/test.py
+96
View File
@@ -0,0 +1,96 @@
import os
from PIL import Image
import torch
import sys
import json
import tqdm
from pathlib import Path
from termcolor import colored
from diffusers import QwenImageEditPipeline
from hacked_models.scheduler import FlowMatchEulerDiscreteScheduler
from hacked_models.pipeline import QwenImageEditPipeline
from hacked_models.models import QwenImageTransformer2DModel
import sys
import json
import tqdm
import argparse
from hacked_models.utils import *
seed_everything(42)
parser = argparse.ArgumentParser()
parser.add_argument("--model_path", type=str, default="Qwen/Qwen-Image-Edit")
parser.add_argument("--data_root", type=str, required=True)
parser.add_argument("--out_path", type=str, required=True)
parser.add_argument("--name", type=str, required=True)
parser.add_argument("--cond_b", type=float, required=True)
parser.add_argument("--cond_delta", type=float, required=True)
args = parser.parse_args()
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
os.path.join(args.model_path, "scheduler"),
torch_dtype=torch.bfloat16,
)
transformer = QwenImageTransformer2DModel.from_pretrained(
os.path.join(args.model_path, "transformer"),
torch_dtype=torch.bfloat16,
)
pipeline = QwenImageEditPipeline.from_pretrained(args.model_path, torch_dtype=torch.bfloat16,
scheduler = scheduler,
transformer=transformer,
)
print("pipeline loaded")
pipeline.to(torch.bfloat16)
pipeline.to("cuda")
pipeline.set_progress_bar_config(disable=None)
DATA_ROOT = args.data_root
data_root = '/'.join(DATA_ROOT.split('/')[:-1])
out_path = os.path.join(args.out_path, DATA_ROOT.split('/')[-2],args.name)
os.makedirs(out_path,exist_ok=True)
with open (DATA_ROOT , 'r') as f:
test_pairs = json.load(f)
print(colored(out_path,color = "green"))
for name , pair in tqdm.tqdm(list(test_pairs.items())):
sample_name = name
img_folder, img_name = '/'.join(pair["image_path"].split('/')[:-1]) , pair["image_path"].split('/')[-1]
# original_prompt = pair["original_prompt"]
# editing_prompt = pair["editing_prompt"]
editing_instruction = pair["editing_instruction"]
input_image = Image.open(os.path.join(data_root,img_folder,img_name)).convert('RGB')
os.makedirs(os.path.join(out_path,img_folder),exist_ok=True)
os.makedirs(os.path.join(out_path,img_folder),exist_ok=True)
if os.path.exists(os.path.join(out_path,img_folder,img_name)):
print('Already generated.')
continue
grid = []
prompt = editing_instruction
inputs = {
"image": input_image,
"prompt": prompt,
"generator": torch.manual_seed(42),
"true_cfg_scale": 4.0,
"negative_prompt": " ",
"num_inference_steps": 24,
"return_dict": False,
"grag_scale":[((512,1.0,1.0),(4096,args.cond_b,args.cond_delta))]*60,
}
with torch.inference_mode():
output = pipeline(**inputs)
image,x0_images,saved_outputs = output
image[0].save(os.path.join(out_path,img_folder,img_name))
Qwen_Edit_GRAG_node.py
+179
View File
@@ -0,0 +1,179 @@
# !/usr/bin/env python
# -*- coding: UTF-8 -*-
import math
import numpy as np
import torch
import os
from diffusers.hooks import apply_group_offloading
import folder_paths
from typing_extensions import override
from comfy_api.latest import ComfyExtension, io
import nodes
import comfy.model_management as mm
from .model_loader_utils import tensor2list,nomarl_upscale,get_emb_data
from .Qwen_Edit_GRAG.inference import load_model,inference
from .Qwen_Edit_GRAG.hacked_models.scheduler import FlowMatchEulerDiscreteScheduler
import node_helpers
MAX_SEED = np.iinfo(np.int32).max
node_cr_path = os.path.dirname(os.path.abspath(__file__))
device = torch.device(
"cuda:0") if torch.cuda.is_available() else torch.device(
"mps") if torch.backends.mps.is_available() else torch.device(
"cpu")
weigths_gguf_current_path = os.path.join(folder_paths.models_dir, "gguf")
if not os.path.exists(weigths_gguf_current_path):
os.makedirs(weigths_gguf_current_path)
folder_paths.add_model_folder_path("gguf", weigths_gguf_current_path) # gguf dir
class Qwen_Edit_GRAG_SM_Model(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="Qwen_Edit_GRAG_SM_Model",
display_name="Qwen_Edit_GRAG_SM_Model",
category="Qwen_Edit_GRAG",
inputs=[
io.Combo.Input("dit",options= ["none"] + folder_paths.get_filename_list("diffusion_models") ),
io.Combo.Input("gguf",options= ["none"] + folder_paths.get_filename_list("gguf") ),
],
outputs=[
io.Custom("Qwen_Edit_GRAG_SM_Model").Output(display_name="model"),
],
)
@classmethod
def execute(cls, dit,gguf) -> io.NodeOutput:
dit_path=folder_paths.get_full_path("diffusion_models", dit) if dit != "none" else None
gguf_path=folder_paths.get_full_path("gguf", gguf) if gguf != "none" else None
pipeline = load_model(gguf_path,dit_path,node_cr_path)
return io.NodeOutput(pipeline)
class Qwen_Edit_GRAG_SM_Encode(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="Qwen_Edit_GRAG_SM_Encode",
display_name="Qwen_Edit_GRAG_SM_Encode",
category="Qwen_Edit_GRAG",
inputs=[
io.Clip.Input("clip"),
io.Vae.Input("vae"),
io.Image.Input("image"),
io.Int.Input("width", default=1024, min=256, max=nodes.MAX_RESOLUTION,step=16,display_mode=io.NumberDisplay.number),
io.Int.Input("height", default=1024, min=256, max=nodes.MAX_RESOLUTION,step=16,display_mode=io.NumberDisplay.number),
io.String.Input("pos_text", multiline=True,default=",best"),
io.String.Input("neg_text", multiline=True,default="bad anatomy, bad hands, missing fingers, extra fingers,three hands, three legs, bad arms, missing legs, missing arms, poorly drawn face, bad face, fused face, cloned face, three crus, fused feet, fused thigh, extra crus, ugly fingers, horn,amputation, disconnected limbs"),
],
outputs=[
io.Conditioning.Output(display_name="positive"),
io.Conditioning.Output(display_name="negative"),
],
)
@classmethod
def execute(cls, clip, vae,image,width,height,pos_text,neg_text,) -> io.NodeOutput:
tensor_list=tensor2list(image,width,height)
pli_image=nomarl_upscale(image,width,height) if isinstance(image,torch.Tensor) else None
postive,ref_latents=get_emb_data(clip,vae,pos_text,tensor_list,)
negative,_=get_emb_data(clip,vae,neg_text,tensor_list,ng=True,img=tensor_list[0] if tensor_list is not None else None )
postive=node_helpers.conditioning_set_values(postive, {"ref_latents": ref_latents})
# gc cf model
cf_models=mm.loaded_models()
try:
for pipe in cf_models:
pipe.unpatch_model(device_to=torch.device("cpu"))
print(f"Unpatching models.{pipe}")
except: pass
mm.soft_empty_cache()
torch.cuda.empty_cache()
max_gpu_memory = torch.cuda.max_memory_allocated()
print(f"After Max GPU memory allocated: {max_gpu_memory / 1000 ** 3:.2f} GB")
return io.NodeOutput(postive,negative)
class Qwen_Edit_GRAG_SM_KSampler(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="Qwen_Edit_GRAG_SM_KSampler",
display_name="Qwen_Edit_GRAG_SM_KSampler",
category="Qwen_Edit_GRAG",
inputs=[
io.Custom("Qwen_Edit_GRAG_SM_Model").Input("model"),
io.Conditioning.Input("positive"),
io.Conditioning.Input("negative"),
io.Combo.Input("lora",options= ["none"] + folder_paths.get_filename_list("loras")),
io.Int.Input("steps", default=24, min=1, max=1024,step=1,display_mode=io.NumberDisplay.number),
io.Float.Input("guidance_scale", default=4.0, min=0, max=20,step=0.01,display_mode=io.NumberDisplay.number),
io.Int.Input("seed", default=0, min=0, max=MAX_SEED,display_mode=io.NumberDisplay.number),
io.Float.Input("cond_b", default=1.0, min=0.01, max=10.0,step=0.01,display_mode=io.NumberDisplay.number),
io.Float.Input("cond_delta", default=1.10, min=0.01, max=10.0,step=0.01,display_mode=io.NumberDisplay.number),
io.Int.Input("block_num", default=10, min=1, max=MAX_SEED,display_mode=io.NumberDisplay.number),
],
outputs=[
io.Latent.Output(display_name="latent"),
io.Latent.Output(display_name="latents"),
],
)
@classmethod
def execute(cls, model,positive,negative,lora,steps,guidance_scale,seed,cond_b,cond_delta,block_num,) -> io.NodeOutput:
adapter_path=folder_paths.get_full_path("loras", lora) if lora != "none" else None
if adapter_path is not None:
model.load_lora_weights(adapter_path,weight_name= os.path.basename(adapter_path))
scheduler_config = {
"base_image_seq_len": 256,
"base_shift": math.log(3), # We use shift=3 in distillation
"invert_sigmas": False,
"max_image_seq_len": 8192,
"max_shift": math.log(3), # We use shift=3 in distillation
"num_train_timesteps": 1000,
"shift": 1.0,
"shift_terminal": None, # set shift_terminal to None
"stochastic_sampling": False,
"time_shift_type": "exponential",
"use_beta_sigmas": False,
"use_dynamic_shifting": True,
"use_exponential_sigmas": False,
"use_karras_sigmas": False,
}
model.scheduler = FlowMatchEulerDiscreteScheduler.from_config(scheduler_config)
# apply offloading
apply_group_offloading(model.transformer, onload_device=torch.device("cuda"), offload_type="block_level", num_blocks_per_group=block_num)
# infer
lat,lats=inference(model,positive,negative,steps,seed,guidance_scale,cond_b,cond_delta)
lats=torch.cat(lats,dim=0)
out_put={"samples":lat}
out_puts={"samples":lats}
return io.NodeOutput(out_put,out_puts)
from aiohttp import web
from server import PromptServer
@PromptServer.instance.routes.get("/Qwen_Edit_GRAG_SM_Extension")
async def get_hello(request):
return web.json_response("Qwen_Edit_GRAG_SM_Extension")
class Qwen_Edit_GRAG_SM_Extension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[io.ComfyNode]]:
return [
Qwen_Edit_GRAG_SM_Model,
Qwen_Edit_GRAG_SM_Encode,
Qwen_Edit_GRAG_SM_KSampler,
]
async def comfy_entrypoint() -> Qwen_Edit_GRAG_SM_Extension: # ComfyUI calls this to load your extension and its nodes.
return Qwen_Edit_GRAG_SM_Extension()
README.md
+32 -1
View File
@@ -1,7 +1,38 @@
# ComfyUI_GRAG_Image_Editing
[GRAG-Image-Editing](https://github.com/little-misfit/GRAG-Image-Editing) : Group-Relative Attention Guidance for Image Editing,you can try it in comfyUI
# Coming soon
1.Installation
-----
In the ./ComfyUI/custom_nodes directory, run the following:
```
git clone https://github.com/smthemex/ComfyUI_GRAG_Image_Editing
```
2.requirements
----
* 不装也行，没什么需求，diffuser版本高点
```
pip install -r requirements.txt
```
3.Model
----
* gguf or transformer [smthem/Qwen-Image-GGUF](https://huggingface.co/smthem/Qwen-Image-GGUF/tree/main) or other or comfy-org optional/随便哪个gguf或者comfyUI官方的transformer
* comfyUI normal： qwen-image vae and qwen_2.5_vl_7b
* lora, lightx2v 8step lora# 千问edit加速
```
├── ComfyUI/models/gguf # or transformer
| ├── Qwen-Image-BF16.gguf # or Q8
├── ComfyUI/models/diffusion_models # or gguf
| ├── Qwen-Image-BF16..safetensors # or e4m3fn
├── ComfyUI/models/vae
| ├─Qwen-Image.safetensors # rename it 换个名字
├── ComfyUI/models/clip
| ├──qwen_2.5_vl_7b_fp8_scaled.safetensors
├── ComfyUI/models/loras
| ├──Qwen-Image-Edit-Lightning-8steps-V1.0-bf16.safetensors
```
# Example
![](https://github.com/smthemex/ComfyUI_GRAG_Image_Editing/blob/main/example_workflows/example.png)
__init__.py
+2
View File
@@ -0,0 +1,2 @@
from .Qwen_Edit_GRAG_node import *
model_loader_utils.py
+422
View File
@@ -0,0 +1,422 @@
# !/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
import torch
import gc
from PIL import Image
import numpy as np
import math
import comfy.utils
import cv2
import folder_paths
from comfy.utils import common_upscale,ProgressBar
from safetensors.torch import load_file
cur_path = os.path.dirname(os.path.abspath(__file__))
def get_emb_data(clip,vae,prompt,image_list,ng=False,img=None,plus=False) : #image_list[ tensor,]
ref_latents = None
if image_list is None :
images = []
elif image_list is not None and not ng:
images,ref_latents=get_image(vae,image_list,plus)
else:
samples = img.movedim(-1, 1)
total = int(1024 * 1024)
scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
width = round(samples.shape[3] * scale_by)
height = round(samples.shape[2] * scale_by)
s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
image = s.movedim(1, -1)
images = [image[:, :, :, :3]]
tokens = clip.tokenize(prompt,images=images)
conditioning = clip.encode_from_tokens_scheduled(tokens,)
return conditioning,ref_latents
def get_image(vae,imgs,plus=False):
ref_latents = None
if plus:
images = []
ref_latents = []
for img in imgs:
samples = img.movedim(-1, 1)
total = int(1024 * 1024)
scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
width = round(samples.shape[3] * scale_by)
height = round(samples.shape[2] * scale_by)
s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
image = s.movedim(1, -1)
images.append(image[:, :, :, :3])
#images = [image[:, :, :, :3]]
if vae is not None:
ref_lat = vae.encode(image[:, :, :, :3])
ref_latents.append(ref_lat)
else:
image=imgs[0]
samples = image.movedim(-1, 1)
total = int(1024 * 1024)
scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
width = round(samples.shape[3] * scale_by)
height = round(samples.shape[2] * scale_by)
s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
image = s.movedim(1, -1)
images = [image[:, :, :, :3]]
if vae is not None:
ref_latents = vae.encode(image[:, :, :, :3])
return images,ref_latents
def encode_image( image, vae):
if image is None:
return None
ref_latents=None
samples = image.movedim(-1, 1)
total = int(1024 * 1024)
scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
width = round(samples.shape[3] * scale_by)
height = round(samples.shape[2] * scale_by)
s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
image = s.movedim(1, -1)
if vae is not None:
ref_latents = vae.encode(image[:, :, :, :3])
return ref_latents
def add_mean(latents):
vae_config={"latents_mean": [
-0.7571,
-0.7089,
-0.9113,
0.1075,
-0.1745,
0.9653,
-0.1517,
1.5508,
0.4134,
-0.0715,
0.5517,
-0.3632,
-0.1922,
-0.9497,
0.2503,
-0.2921
],
"latents_std": [
2.8184,
1.4541,
2.3275,
2.6558,
1.2196,
1.7708,
2.6052,
2.0743,
3.2687,
2.1526,
2.8652,
1.5579,
1.6382,
1.1253,
2.8251,
1.916
],}
latents_mean = (torch.tensor(vae_config["latents_mean"]).view(1, 16, 1, 1, 1).to(latents.device, latents.dtype))
latents_std = 1.0 / torch.tensor(vae_config["latents_std"]).view(1, 16, 1, 1, 1).to(latents.device, latents.dtype)
latents = latents / latents_std + latents_mean
image_latent_height, image_latent_width = latents.shape[3:]
image_latents = pack_latents_(
latents, 1, 16, image_latent_height, image_latent_width)
return image_latents
def pack_latents_(latents, batch_size, num_channels_latents, height, width):
latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
latents = latents.permute(0, 2, 4, 1, 3, 5)
latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
return latents
def load_lora(model, lora_1, lora_2, lora_scale1, lora_scale2):
lora_path_1=folder_paths.get_full_path("loras", lora_1) if lora_1 != "none" else None
lora_path_2=folder_paths.get_full_path("loras", lora_2) if lora_2 != "none" else None
# lora_list=[i for i in [lora_path_1,lora_path_2] if i is not None]
# lora_scales=[lora_scale1,lora_scale2]
all_adapters = model.get_list_adapters()
dit_list=[]
if all_adapters:
dit_list= all_adapters.get('transformer',[])+all_adapters.get('transformer_2',[])
if lora_path_1 is not None:
adapter_name=os.path.splitext(os.path.basename(lora_path_1))[0].replace(".", "_")
dit_list2=all_adapters.get('transformer_2',[])
if dit_list2:
if adapter_name in dit_list: #dit_list
pass
else:
for i in dit_list2:
model.delete_adapters(i)
print(f"去除dit中未加载的lora: {i}")
try:
model.load_lora_weights(lora_path_1, adapter_name=adapter_name,**{"load_into_transformer_2": True})
model.set_adapters([adapter_name], adapter_weights=lora_scale1)
except KeyError as e:
try:
print(f"检测到特殊的 LoRA 格式，尝试手动处理: {lora_path_1}")
state_dict = torch.load(lora_path_1, map_location="cpu",weights_only=False) if not lora_path_1.endswith(".safetensors") else load_file(lora_path_1,)
processed_state_dict = preprocess_lora_state_dict(state_dict)
model.load_lora_weights(processed_state_dict, adapter_name=adapter_name,**{"load_into_transformer_2": True})
model.set_adapters([adapter_name], adapter_weights=lora_scale1)
except:
print(f"加载LoRA权重失败: {e}")
pass
else:
try:
model.load_lora_weights(lora_path_1, adapter_name=adapter_name,**{"load_into_transformer_2": True})
model.set_adapters([adapter_name], adapter_weights=lora_scale1)
except KeyError as e:
try:
print(f"检测到特殊的 LoRA 格式，尝试手动处理: {lora_path_1}")
state_dict = torch.load(lora_path_1, map_location="cpu",weights_only=False) if not lora_path_1.endswith(".safetensors") else load_file(lora_path_1,)
processed_state_dict = preprocess_lora_state_dict(state_dict)
model.load_lora_weights(processed_state_dict, adapter_name=adapter_name,**{"load_into_transformer_2": True})
model.set_adapters([adapter_name], adapter_weights=lora_scale1)
del processed_state_dict
except:
print(f"加载LoRA权重失败: {e}")
pass
if lora_path_2 is not None:
adapter_name=os.path.splitext(os.path.basename(lora_path_2))[0].replace(".", "_")
dit_list=all_adapters.get('transformer',[])
if dit_list:
if adapter_name in dit_list: #dit_list
pass
else:
for i in dit_list:
model.delete_adapters(i)
print(f"去除dit中未加载的lora: {i}")
try:
model.load_lora_weights(lora_path_2, adapter_name=adapter_name,**{"load_into_transformer_2": False})
model.set_adapters([adapter_name], adapter_weights=lora_scale2)
except KeyError as e:
try:
print(f"检测到特殊的 LoRA 格式，尝试手动处理: {lora_path_2}")
state_dict = torch.load(lora_path_2, map_location="cpu",weights_only=False) if not lora_path_2.endswith(".safetensors") else load_file(lora_path_2,)
processed_state_dict = preprocess_lora_state_dict(state_dict)
model.load_lora_weights(processed_state_dict, adapter_name=adapter_name,**{"load_into_transformer_2": False})
model.set_adapters([adapter_name], adapter_weights=lora_scale2)
del processed_state_dict
except:
print(f"加载LoRA权重失败: {e}")
pass
else:
try:
model.load_lora_weights(lora_path_2, adapter_name=adapter_name,**{"load_into_transformer_2": False})
model.set_adapters([adapter_name], adapter_weights=lora_scale2)
except KeyError as e:
try:
print(f"检测到特殊的 LoRA 格式，尝试手动处理: {lora_path_2}")
state_dict = torch.load(lora_path_2, map_location="cpu",weights_only=False) if not lora_path_2.endswith(".safetensors") else load_file(lora_path_2,)
processed_state_dict = preprocess_lora_state_dict(state_dict)
model.load_lora_weights(processed_state_dict, adapter_name=adapter_name,**{"load_into_transformer_2": False})
model.set_adapters([adapter_name], adapter_weights=lora_scale2)
del processed_state_dict
except:
print(f"加载LoRA权重失败: {e}")
pass
return model
def preprocess_lora_state_dict(state_dict):
processed_dict = state_dict.copy()
keys_to_remove = [
'head.head.diff_b',
'head.head.diff_m',
'head.head.diff',
'patch_embedding.diff',
'patch_embedding.diff_b',
'blocks.*.diff_m', # 匹配所有blocks的diff_m
'head.head.lora_down'
'diffusion_model.head.head.diff'
'diffusion_model.head.head.diff_b'
'diffusion_model.head.lora_down'
]
keys_to_delete = []
for key in processed_dict.keys():
if key.endswith('.diff_m'):
keys_to_delete.append(key)
for key in keys_to_delete:
processed_dict.pop(key, None)
print(f"移除键: {key}")
for key in keys_to_remove:
if key in processed_dict:
processed_dict.pop(key, None)
print(f"移除键: {key}")
return processed_dict
def gc_cleanup():
gc.collect()
torch.cuda.empty_cache()
def tensor2cv(tensor_image):
if len(tensor_image.shape)==4:# b hwc to hwc
tensor_image=tensor_image.squeeze(0)
if tensor_image.is_cuda:
tensor_image = tensor_image.cpu()
tensor_image=tensor_image.numpy()
#反归一化
maxValue=tensor_image.max()
tensor_image=tensor_image*255/maxValue
img_cv2=np.uint8(tensor_image)#32 to uint8
img_cv2=cv2.cvtColor(img_cv2,cv2.COLOR_RGB2BGR)
return img_cv2
def phi2narry(img):
img = torch.from_numpy(np.array(img).astype(np.float32) / 255.0).unsqueeze(0)
return img
def tensor2image(tensor):
tensor = tensor.cpu()
image_np = tensor.squeeze().mul(255).clamp(0, 255).byte().numpy()
image = Image.fromarray(image_np, mode='RGB')
return image
def tensor2pillist(tensor_in):
d1, _, _, _ = tensor_in.size()
if d1 == 1:
img_list = [tensor2image(tensor_in)]
else:
tensor_list = torch.chunk(tensor_in, chunks=d1)
img_list=[tensor2image(i) for i in tensor_list]
return img_list
def tensor2pillist_upscale(tensor_in,width,height):
d1, _, _, _ = tensor_in.size()
if d1 == 1:
img_list = [nomarl_upscale(tensor_in,width,height)]
else:
tensor_list = torch.chunk(tensor_in, chunks=d1)
img_list=[nomarl_upscale(i,width,height) for i in tensor_list]
return img_list
def tensor2list(tensor_in,width,height):
if tensor_in is None:
return None
d1, _, _, _ = tensor_in.size()
if d1 == 1:
tensor_list = [tensor_upscale(tensor_in,width,height)]
else:
tensor_list_ = torch.chunk(tensor_in, chunks=d1)
tensor_list=[tensor_upscale(i,width,height) for i in tensor_list_]
return tensor_list
def tensor_upscale(tensor, width, height):
samples = tensor.movedim(-1, 1)
samples = common_upscale(samples, width, height, "bilinear", "center")
samples = samples.movedim(1, -1)
return samples
def nomarl_upscale(img, width, height):
samples = img.movedim(-1, 1)
img = common_upscale(samples, width, height, "bilinear", "center")
samples = img.movedim(1, -1)
img = tensor2image(samples)
return img
def cv2tensor(img,bgr2rgb=True):
assert type(img) == np.ndarray, 'the img type is {}, but ndarry expected'.format(type(img))
if bgr2rgb:
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = torch.from_numpy(img.transpose((2, 0, 1)))
return img.float().div(255).permute(1, 2, 0).unsqueeze(0)
def images_generator(img_list: list, ):
# get img size
sizes = {}
for image_ in img_list:
if isinstance(image_, Image.Image):
count = sizes.get(image_.size, 0)
sizes[image_.size] = count + 1
elif isinstance(image_, np.ndarray):
count = sizes.get(image_.shape[:2][::-1], 0)
sizes[image_.shape[:2][::-1]] = count + 1
else:
raise "unsupport image list,must be pil or cv2!!!"
size = max(sizes.items(), key=lambda x: x[1])[0]
yield size[0], size[1]
# any to tensor
def load_image(img_in):
if isinstance(img_in, Image.Image):
img_in = img_in.convert("RGB")
i = np.array(img_in, dtype=np.float32)
i = torch.from_numpy(i).div_(255)
if i.shape[0] != size[1] or i.shape[1] != size[0]:
i = torch.from_numpy(i).movedim(-1, 0).unsqueeze(0)
i = common_upscale(i, size[0], size[1], "lanczos", "center")
i = i.squeeze(0).movedim(0, -1).numpy()
return i
elif isinstance(img_in, np.ndarray):
i = cv2.cvtColor(img_in, cv2.COLOR_BGR2RGB).astype(np.float32)
i = torch.from_numpy(i).div_(255)
print(i.shape)
return i
else:
raise "unsupport image list,must be pil,cv2 or tensor!!!"
total_images = len(img_list)
processed_images = 0
pbar = ProgressBar(total_images)
images = map(load_image, img_list)
try:
prev_image = next(images)
while True:
next_image = next(images)
yield prev_image
processed_images += 1
pbar.update_absolute(processed_images, total_images)
prev_image = next_image
except StopIteration:
pass
if prev_image is not None:
yield prev_image
def load_images_list(img_list: list, ):
gen = images_generator(img_list)
(width, height) = next(gen)
images = torch.from_numpy(np.fromiter(gen, np.dtype((np.float32, (height, width, 3)))))
if len(images) == 0:
raise FileNotFoundError(f"No images could be loaded .")
return images
def get_video_files(directory, extensions=None):
if extensions is None:
extensions = ['webm', 'mp4', 'mkv', 'gif', 'mov']
extensions = [ext.lower() for ext in extensions]
video_files = []
for root, dirs, files in os.walk(directory):
for file in files:
_, ext = os.path.splitext(file)
ext = ext.lower()[1:]
if ext in extensions:
full_path = os.path.join(root, file)
video_files.append(full_path)
return video_files
pyproject.toml
+16
View File
@@ -0,0 +1,16 @@
[project]
name = "grag_image_editing"
description = "GRAG-Image-Editing : Group-Relative Attention Guidance for Image Editing,you can try it in comfyUI"
version = "1.0.0"
license = {file = "LICENSE"}
dependencies = ["accelerate", "diffusers", "ipykernel", "matplotlib", "pathlib", "pickleshare", "pip-chill", "termcolor", "tomli", "torch", "torchvision", "transformers", "huggingface_hub"]
[project.urls]
Repository = "https://github.com/smthemex/ComfyUI_GRAG_Image_Editing"
# Used by Comfy Registry https://registry.comfy.org
[tool.comfy]
PublisherId = "smthemex"
DisplayName = "ComfyUI_GRAG_Image_Editing"
Icon = ""
includes = []
requirements.txt
+13
View File
@@ -0,0 +1,13 @@
accelerate
diffusers
ipykernel
matplotlib
pathlib
pickleshare
pip-chill
termcolor
tomli
torch
torchvision
transformers
huggingface_hub