init
This commit is contained in:
smthemex
@@ -0,0 +1,2 @@
|
||||
__pycache__/
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"_class_name": "QwenImageEditPipeline",
|
||||
"_diffusers_version": "0.35.0.dev0",
|
||||
"processor": [
|
||||
"transformers",
|
||||
"Qwen2VLProcessor"
|
||||
],
|
||||
"scheduler": [
|
||||
"diffusers",
|
||||
"FlowMatchEulerDiscreteScheduler"
|
||||
],
|
||||
"text_encoder": [
|
||||
"transformers",
|
||||
"Qwen2_5_VLForConditionalGeneration"
|
||||
],
|
||||
"tokenizer": [
|
||||
"transformers",
|
||||
"Qwen2Tokenizer"
|
||||
],
|
||||
"transformer": [
|
||||
"diffusers",
|
||||
"QwenImageTransformer2DModel"
|
||||
],
|
||||
"vae": [
|
||||
"diffusers",
|
||||
"AutoencoderKLQwenImage"
|
||||
]
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
{
|
||||
"</tool_call>": 151658,
|
||||
"<tool_call>": 151657,
|
||||
"<|box_end|>": 151649,
|
||||
"<|box_start|>": 151648,
|
||||
"<|endoftext|>": 151643,
|
||||
"<|file_sep|>": 151664,
|
||||
"<|fim_middle|>": 151660,
|
||||
"<|fim_pad|>": 151662,
|
||||
"<|fim_prefix|>": 151659,
|
||||
"<|fim_suffix|>": 151661,
|
||||
"<|im_end|>": 151645,
|
||||
"<|im_start|>": 151644,
|
||||
"<|image_pad|>": 151655,
|
||||
"<|object_ref_end|>": 151647,
|
||||
"<|object_ref_start|>": 151646,
|
||||
"<|quad_end|>": 151651,
|
||||
"<|quad_start|>": 151650,
|
||||
"<|repo_name|>": 151663,
|
||||
"<|video_pad|>": 151656,
|
||||
"<|vision_end|>": 151653,
|
||||
"<|vision_pad|>": 151654,
|
||||
"<|vision_start|>": 151652
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
|
||||
You are a helpful assistant.<|im_end|>
|
||||
{% endif %}<|im_start|>{{ message['role'] }}
|
||||
{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
|
||||
{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
|
||||
{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
|
||||
{% endif %}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,37 +0,0 @@
|
||||
{
|
||||
"crop_size": null,
|
||||
"data_format": "channels_first",
|
||||
"default_to_square": true,
|
||||
"device": null,
|
||||
"disable_grouping": null,
|
||||
"do_center_crop": null,
|
||||
"do_convert_rgb": true,
|
||||
"do_normalize": true,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"image_mean": [
|
||||
0.48145466,
|
||||
0.4578275,
|
||||
0.40821073
|
||||
],
|
||||
"image_processor_type": "Qwen2VLImageProcessorFast",
|
||||
"image_std": [
|
||||
0.26862954,
|
||||
0.26130258,
|
||||
0.27577711
|
||||
],
|
||||
"input_data_format": null,
|
||||
"max_pixels": 12845056,
|
||||
"merge_size": 2,
|
||||
"min_pixels": 3136,
|
||||
"patch_size": 14,
|
||||
"processor_class": "Qwen2VLProcessor",
|
||||
"resample": 3,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"return_tensors": null,
|
||||
"size": {
|
||||
"longest_edge": 12845056,
|
||||
"shortest_edge": 3136
|
||||
},
|
||||
"temporal_patch_size": 2
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,208 +0,0 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151646": {
|
||||
"content": "<|object_ref_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151647": {
|
||||
"content": "<|object_ref_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151648": {
|
||||
"content": "<|box_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151649": {
|
||||
"content": "<|box_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151650": {
|
||||
"content": "<|quad_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151651": {
|
||||
"content": "<|quad_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151652": {
|
||||
"content": "<|vision_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151653": {
|
||||
"content": "<|vision_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151654": {
|
||||
"content": "<|vision_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151655": {
|
||||
"content": "<|image_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151656": {
|
||||
"content": "<|video_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151657": {
|
||||
"content": "<tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151658": {
|
||||
"content": "</tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151659": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151660": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151661": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151662": {
|
||||
"content": "<|fim_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151663": {
|
||||
"content": "<|repo_name|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151664": {
|
||||
"content": "<|file_sep|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": {},
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"processor_class": "Qwen2VLProcessor",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
{
|
||||
"crop_size": null,
|
||||
"data_format": "channels_first",
|
||||
"default_to_square": true,
|
||||
"device": null,
|
||||
"do_center_crop": null,
|
||||
"do_convert_rgb": true,
|
||||
"do_normalize": true,
|
||||
"do_pad": null,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"do_sample_frames": false,
|
||||
"fps": null,
|
||||
"image_mean": [
|
||||
0.48145466,
|
||||
0.4578275,
|
||||
0.40821073
|
||||
],
|
||||
"image_std": [
|
||||
0.26862954,
|
||||
0.26130258,
|
||||
0.27577711
|
||||
],
|
||||
"input_data_format": null,
|
||||
"max_frames": 768,
|
||||
"max_pixels": 12845056,
|
||||
"merge_size": 2,
|
||||
"min_frames": 4,
|
||||
"min_pixels": 3136,
|
||||
"num_frames": null,
|
||||
"patch_size": 14,
|
||||
"processor_class": "Qwen2VLProcessor",
|
||||
"resample": 3,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"size": {
|
||||
"longest_edge": 12845056,
|
||||
"shortest_edge": 3136
|
||||
},
|
||||
"size_divisor": null,
|
||||
"temporal_patch_size": 2,
|
||||
"video_metadata": null,
|
||||
"video_processor_type": "Qwen2VLVideoProcessor"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"_class_name": "FlowMatchEulerDiscreteScheduler",
|
||||
"_diffusers_version": "0.35.0.dev0",
|
||||
"base_image_seq_len": 256,
|
||||
"base_shift": 0.5,
|
||||
"invert_sigmas": false,
|
||||
"max_image_seq_len": 8192,
|
||||
"max_shift": 0.9,
|
||||
"num_train_timesteps": 1000,
|
||||
"shift": 1.0,
|
||||
"shift_terminal": 0.02,
|
||||
"stochastic_sampling": false,
|
||||
"time_shift_type": "exponential",
|
||||
"use_beta_sigmas": false,
|
||||
"use_dynamic_shifting": true,
|
||||
"use_exponential_sigmas": false,
|
||||
"use_karras_sigmas": false
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2_5_VLForConditionalGeneration"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 3584,
|
||||
"image_token_id": 151655,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 18944,
|
||||
"max_position_embeddings": 128000,
|
||||
"max_window_layers": 28,
|
||||
"model_type": "qwen2_5_vl",
|
||||
"num_attention_heads": 28,
|
||||
"num_hidden_layers": 28,
|
||||
"num_key_value_heads": 4,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": {
|
||||
"mrope_section": [
|
||||
16,
|
||||
24,
|
||||
24
|
||||
],
|
||||
"rope_type": "default",
|
||||
"type": "default"
|
||||
},
|
||||
"rope_theta": 1000000.0,
|
||||
"sliding_window": 32768,
|
||||
"text_config": {
|
||||
"architectures": [
|
||||
"Qwen2_5_VLForConditionalGeneration"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 3584,
|
||||
"image_token_id": null,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 18944,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 128000,
|
||||
"max_window_layers": 28,
|
||||
"model_type": "qwen2_5_vl_text",
|
||||
"num_attention_heads": 28,
|
||||
"num_hidden_layers": 28,
|
||||
"num_key_value_heads": 4,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": {
|
||||
"mrope_section": [
|
||||
16,
|
||||
24,
|
||||
24
|
||||
],
|
||||
"rope_type": "default",
|
||||
"type": "default"
|
||||
},
|
||||
"rope_theta": 1000000.0,
|
||||
"sliding_window": null,
|
||||
"torch_dtype": "float32",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"video_token_id": null,
|
||||
"vision_end_token_id": 151653,
|
||||
"vision_start_token_id": 151652,
|
||||
"vision_token_id": 151654,
|
||||
"vocab_size": 152064
|
||||
},
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.55.2",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"video_token_id": 151656,
|
||||
"vision_config": {
|
||||
"depth": 32,
|
||||
"fullatt_block_indexes": [
|
||||
7,
|
||||
15,
|
||||
23,
|
||||
31
|
||||
],
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 1280,
|
||||
"in_channels": 3,
|
||||
"in_chans": 3,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 3420,
|
||||
"model_type": "qwen2_5_vl",
|
||||
"num_heads": 16,
|
||||
"out_hidden_size": 3584,
|
||||
"patch_size": 14,
|
||||
"spatial_merge_size": 2,
|
||||
"spatial_patch_size": 14,
|
||||
"temporal_patch_size": 2,
|
||||
"tokens_per_second": 2,
|
||||
"torch_dtype": "float32",
|
||||
"window_size": 112
|
||||
},
|
||||
"vision_end_token_id": 151653,
|
||||
"vision_start_token_id": 151652,
|
||||
"vision_token_id": 151654,
|
||||
"vocab_size": 152064
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"bos_token_id": 151643,
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.05,
|
||||
"temperature": 0.1,
|
||||
"top_k": 1,
|
||||
"top_p": 0.001,
|
||||
"transformers_version": "4.55.2"
|
||||
}
|
||||
@@ -1,737 +0,0 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 8292166656,
|
||||
"total_size": 16584333312
|
||||
},
|
||||
"weight_map": {
|
||||
"lm_head.weight": "model-00004-of-00004.safetensors",
|
||||
"model.embed_tokens.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.norm.weight": "model-00004-of-00004.safetensors",
|
||||
"visual.blocks.0.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.1.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.11.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.12.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.14.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.15.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.16.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.18.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.19.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.2.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.21.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.22.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.23.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.24.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.27.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.28.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.29.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.30.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.31.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.5.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.6.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.8.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.blocks.9.norm2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.merger.ln_q.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.merger.mlp.0.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.merger.mlp.0.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.merger.mlp.2.bias": "model-00001-of-00004.safetensors",
|
||||
"visual.merger.mlp.2.weight": "model-00001-of-00004.safetensors",
|
||||
"visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors"
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
{
|
||||
"</tool_call>": 151658,
|
||||
"<tool_call>": 151657,
|
||||
"<|box_end|>": 151649,
|
||||
"<|box_start|>": 151648,
|
||||
"<|endoftext|>": 151643,
|
||||
"<|file_sep|>": 151664,
|
||||
"<|fim_middle|>": 151660,
|
||||
"<|fim_pad|>": 151662,
|
||||
"<|fim_prefix|>": 151659,
|
||||
"<|fim_suffix|>": 151661,
|
||||
"<|im_end|>": 151645,
|
||||
"<|im_start|>": 151644,
|
||||
"<|image_pad|>": 151655,
|
||||
"<|object_ref_end|>": 151647,
|
||||
"<|object_ref_start|>": 151646,
|
||||
"<|quad_end|>": 151651,
|
||||
"<|quad_start|>": 151650,
|
||||
"<|repo_name|>": 151663,
|
||||
"<|video_pad|>": 151656,
|
||||
"<|vision_end|>": 151653,
|
||||
"<|vision_pad|>": 151654,
|
||||
"<|vision_start|>": 151652
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] }}
|
||||
{%- else %}
|
||||
{{- 'You are a helpful assistant.' }}
|
||||
{%- endif %}
|
||||
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content %}
|
||||
{{- '\n' + message.content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -1,207 +0,0 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151646": {
|
||||
"content": "<|object_ref_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151647": {
|
||||
"content": "<|object_ref_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151648": {
|
||||
"content": "<|box_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151649": {
|
||||
"content": "<|box_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151650": {
|
||||
"content": "<|quad_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151651": {
|
||||
"content": "<|quad_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151652": {
|
||||
"content": "<|vision_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151653": {
|
||||
"content": "<|vision_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151654": {
|
||||
"content": "<|vision_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151655": {
|
||||
"content": "<|image_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151656": {
|
||||
"content": "<|video_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151657": {
|
||||
"content": "<tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151658": {
|
||||
"content": "</tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151659": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151660": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151661": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151662": {
|
||||
"content": "<|fim_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151663": {
|
||||
"content": "<|repo_name|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151664": {
|
||||
"content": "<|file_sep|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": {},
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"_class_name": "QwenImageTransformer2DModel",
|
||||
"_diffusers_version": "0.35.0.dev0",
|
||||
"attention_head_dim": 128,
|
||||
"axes_dims_rope": [
|
||||
16,
|
||||
56,
|
||||
56
|
||||
],
|
||||
"guidance_embeds": false,
|
||||
"in_channels": 64,
|
||||
"joint_attention_dim": 3584,
|
||||
"num_attention_heads": 24,
|
||||
"num_layers": 60,
|
||||
"out_channels": 16,
|
||||
"patch_size": 2
|
||||
}
|
||||
-1940
File diff suppressed because it is too large
Load Diff
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"_class_name": "AutoencoderKLQwenImage",
|
||||
"_diffusers_version": "0.35.0.dev0",
|
||||
"attn_scales": [],
|
||||
"base_dim": 96,
|
||||
"dim_mult": [
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
4
|
||||
],
|
||||
"dropout": 0.0,
|
||||
"latents_mean": [
|
||||
-0.7571,
|
||||
-0.7089,
|
||||
-0.9113,
|
||||
0.1075,
|
||||
-0.1745,
|
||||
0.9653,
|
||||
-0.1517,
|
||||
1.5508,
|
||||
0.4134,
|
||||
-0.0715,
|
||||
0.5517,
|
||||
-0.3632,
|
||||
-0.1922,
|
||||
-0.9497,
|
||||
0.2503,
|
||||
-0.2921
|
||||
],
|
||||
"latents_std": [
|
||||
2.8184,
|
||||
1.4541,
|
||||
2.3275,
|
||||
2.6558,
|
||||
1.2196,
|
||||
1.7708,
|
||||
2.6052,
|
||||
2.0743,
|
||||
3.2687,
|
||||
2.1526,
|
||||
2.8652,
|
||||
1.5579,
|
||||
1.6382,
|
||||
1.1253,
|
||||
2.8251,
|
||||
1.916
|
||||
],
|
||||
"num_res_blocks": 2,
|
||||
"temperal_downsample": [
|
||||
false,
|
||||
true,
|
||||
true
|
||||
],
|
||||
"z_dim": 16
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
"""
|
||||
Minimal Gradio wrapper for the given Qwen-Image-Edit inference script.
|
||||
|
||||
Features:
|
||||
- Loads the model once and reuses it.
|
||||
- Inputs: image, edit prompt, cond_b, cond_delta, optional model path.
|
||||
- Matches your original settings (size 1024, steps=24, true_cfg_scale=4.0,
|
||||
fixed seed=42, and the same GRAG scale structure repeated 60 times).
|
||||
|
||||
Run:
|
||||
pip install gradio pillow torch
|
||||
# plus your project deps providing hacked_models/* and model weights
|
||||
python gradio_qwen_edit_minimal.py
|
||||
|
||||
Then open the local URL printed by Gradio.
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
import gradio as gr
|
||||
import torch
|
||||
from PIL import Image
|
||||
from huggingface_hub import snapshot_download
|
||||
import os
|
||||
# --- your project imports (as in the original script) ---
|
||||
from hacked_models.scheduler import FlowMatchEulerDiscreteScheduler
|
||||
from hacked_models.pipeline import QwenImageEditPipeline
|
||||
from hacked_models.models import QwenImageTransformer2DModel
|
||||
from hacked_models.utils import seed_everything
|
||||
|
||||
# -----------------------------
|
||||
# Global state
|
||||
# -----------------------------
|
||||
_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
_DTYPE = torch.bfloat16 if _DEVICE == "cuda" else torch.float32
|
||||
_PIPELINE: Optional[QwenImageEditPipeline] = None
|
||||
_LOADED_MODEL_PATH: Optional[str] = None
|
||||
|
||||
|
||||
def _load_pipeline(model_path: str) -> QwenImageEditPipeline:
|
||||
"""Load (or reuse) the pipeline for the given model_path."""
|
||||
global _PIPELINE, _LOADED_MODEL_PATH
|
||||
if _PIPELINE is not None and _LOADED_MODEL_PATH == model_path:
|
||||
return _PIPELINE
|
||||
|
||||
# Set seed once (matches original)
|
||||
seed_everything(42)
|
||||
|
||||
# Load components
|
||||
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
|
||||
os.path.join(model_path, "scheduler"), torch_dtype=_DTYPE
|
||||
)
|
||||
transformer = QwenImageTransformer2DModel.from_pretrained(
|
||||
os.path.join(model_path, "transformer"), torch_dtype=_DTYPE
|
||||
)
|
||||
|
||||
pipe = QwenImageEditPipeline.from_pretrained(
|
||||
model_path, torch_dtype=_DTYPE, scheduler=scheduler, transformer=transformer
|
||||
)
|
||||
|
||||
pipe.set_progress_bar_config(disable=None)
|
||||
pipe.to(_DTYPE)
|
||||
pipe.to(_DEVICE)
|
||||
|
||||
_PIPELINE = pipe
|
||||
_LOADED_MODEL_PATH = model_path
|
||||
return pipe
|
||||
|
||||
|
||||
def _build_grag_scale(cond_b: float, cond_delta: float, repeats: int = 60):
|
||||
"""Replicates your original GRAG schedule structure.
|
||||
|
||||
Each element is: ((512, 1.0, 1.0), (4096, cond_b, cond_delta))
|
||||
"""
|
||||
return [((512, 1.0, 1.0), (4096, cond_b, cond_delta))] * repeats
|
||||
|
||||
|
||||
def predict(
|
||||
image: Image.Image,
|
||||
edit_prompt: str,
|
||||
cond_b: float,
|
||||
cond_delta: float,
|
||||
pipeline,
|
||||
):
|
||||
if image is None or not edit_prompt:
|
||||
return None
|
||||
|
||||
|
||||
# Match original preprocessing
|
||||
input_image = image.convert("RGB").resize((1024, 1024))
|
||||
|
||||
inputs = {
|
||||
"image": input_image,
|
||||
"prompt": edit_prompt,
|
||||
"generator": torch.manual_seed(42),
|
||||
"true_cfg_scale": 4.0,
|
||||
"negative_prompt": " ",
|
||||
"num_inference_steps": 24,
|
||||
"return_dict": False,
|
||||
"grag_scale": _build_grag_scale(cond_b, cond_delta, repeats=60),
|
||||
}
|
||||
|
||||
with torch.inference_mode():
|
||||
image_batch, x0_images, saved_outputs = pipe(**inputs)
|
||||
|
||||
# Return the first image (same as original save behavior)
|
||||
return image_batch[0]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
model_dir = "Qwen-Image-Edit"
|
||||
repo_id = "Qwen/Qwen-Image-Edit"
|
||||
|
||||
if not os.path.exists(model_dir) or not os.listdir(model_dir):
|
||||
snapshot_download(repo_id=repo_id, local_dir=model_dir, local_dir_use_symlinks=False)
|
||||
print(f"Model downloaded to {model_dir}")
|
||||
else:
|
||||
print(f"Model already exists at {model_dir}")
|
||||
|
||||
|
||||
|
||||
pipe = _load_pipeline(model_dir)
|
||||
|
||||
|
||||
with gr.Blocks(title="Qwen Image Edit — Minimal GRAG Demo") as demo:
|
||||
gr.Markdown("# Qwen Image Edit — Minimal GRAG Demo\nUpload an image, enter your edit instruction, and set GRAG params.")
|
||||
|
||||
|
||||
with gr.Row():
|
||||
in_image = gr.Image(label="Input Image", type="pil")
|
||||
out_image = gr.Image(label="Edited Output", type="pil")
|
||||
|
||||
edit_prompt = gr.Textbox(label="Edit Instruction", placeholder="e.g., Put a pair of black-framed glasses on him.")
|
||||
with gr.Row():
|
||||
cond_b = gr.Slider(label="cond_b", minimum=0.8, maximum=2.0, value=1.0, step=0.01)
|
||||
cond_delta = gr.Slider(label="cond_delta", minimum=0.8, maximum=2.0, value=1.0, step=0.01)
|
||||
|
||||
run_btn = gr.Button("Run Edit")
|
||||
|
||||
run_btn.click(
|
||||
fn=predict,
|
||||
inputs=[in_image, edit_prompt, cond_b, cond_delta, pipe],
|
||||
outputs=[out_image],
|
||||
api_name="run_edit",
|
||||
)
|
||||
|
||||
gr.Markdown(
|
||||
"""
|
||||
**Notes**
|
||||
- Uses fixed seed=42 and num_inference_steps=24 to match your script.
|
||||
- Resizes the input to 1024×1024 before inference (as in your code).
|
||||
- `grag_scale` is built as a list of length 60 with the same tuples.
|
||||
- Automatically chooses CUDA if available; otherwise runs on CPU.
|
||||
"""
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
demo.queue().launch(share=True)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,686 +0,0 @@
|
||||
# Copyright 2025 Qwen-Image Team, The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import functools
|
||||
import math
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin
|
||||
from diffusers.utils import USE_PEFT_BACKEND, logging, scale_lora_layers, unscale_lora_layers
|
||||
from diffusers.utils.torch_utils import maybe_allow_in_graph
|
||||
from diffusers.models.attention import AttentionMixin, FeedForward
|
||||
from diffusers.models.attention_dispatch import dispatch_attention_fn
|
||||
from diffusers.models.attention_processor import Attention
|
||||
from diffusers.models.cache_utils import CacheMixin
|
||||
from diffusers.models.embeddings import TimestepEmbedding, Timesteps
|
||||
from diffusers.models.modeling_outputs import Transformer2DModelOutput
|
||||
from diffusers.models.modeling_utils import ModelMixin
|
||||
from diffusers.models.normalization import AdaLayerNormContinuous, RMSNorm
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
|
||||
def _lastdim_l2norm(x: torch.Tensor, eps: float = 1e-12):
|
||||
# 返回最后一维的 L2 范数，保持维度
|
||||
return
|
||||
|
||||
|
||||
def get_timestep_embedding(
|
||||
timesteps: torch.Tensor,
|
||||
embedding_dim: int,
|
||||
flip_sin_to_cos: bool = False,
|
||||
downscale_freq_shift: float = 1,
|
||||
scale: float = 1,
|
||||
max_period: int = 10000,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings.
|
||||
|
||||
Args
|
||||
timesteps (torch.Tensor):
|
||||
a 1-D Tensor of N indices, one per batch element. These may be fractional.
|
||||
embedding_dim (int):
|
||||
the dimension of the output.
|
||||
flip_sin_to_cos (bool):
|
||||
Whether the embedding order should be `cos, sin` (if True) or `sin, cos` (if False)
|
||||
downscale_freq_shift (float):
|
||||
Controls the delta between frequencies between dimensions
|
||||
scale (float):
|
||||
Scaling factor applied to the embeddings.
|
||||
max_period (int):
|
||||
Controls the maximum frequency of the embeddings
|
||||
Returns
|
||||
torch.Tensor: an [N x dim] Tensor of positional embeddings.
|
||||
"""
|
||||
assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"
|
||||
|
||||
half_dim = embedding_dim // 2
|
||||
exponent = -math.log(max_period) * torch.arange(
|
||||
start=0, end=half_dim, dtype=torch.float32, device=timesteps.device
|
||||
)
|
||||
exponent = exponent / (half_dim - downscale_freq_shift)
|
||||
|
||||
emb = torch.exp(exponent).to(timesteps.dtype)
|
||||
emb = timesteps[:, None].float() * emb[None, :]
|
||||
|
||||
# scale embeddings
|
||||
emb = scale * emb
|
||||
|
||||
# concat sine and cosine embeddings
|
||||
emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1)
|
||||
|
||||
# flip sine and cosine embeddings
|
||||
if flip_sin_to_cos:
|
||||
emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)
|
||||
|
||||
# zero pad
|
||||
if embedding_dim % 2 == 1:
|
||||
emb = torch.nn.functional.pad(emb, (0, 1, 0, 0))
|
||||
return emb
|
||||
|
||||
|
||||
def apply_rotary_emb_qwen(
|
||||
x: torch.Tensor,
|
||||
freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]],
|
||||
use_real: bool = True,
|
||||
use_real_unbind_dim: int = -1,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings
|
||||
to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are
|
||||
reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting
|
||||
tensors contain rotary embeddings and are returned as real tensors.
|
||||
|
||||
Args:
|
||||
x (`torch.Tensor`):
|
||||
Query or key tensor to apply rotary embeddings. [B, S, H, D] xk (torch.Tensor): Key tensor to apply
|
||||
freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],)
|
||||
|
||||
Returns:
|
||||
Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings.
|
||||
"""
|
||||
if use_real:
|
||||
cos, sin = freqs_cis # [S, D]
|
||||
cos = cos[None, None]
|
||||
sin = sin[None, None]
|
||||
cos, sin = cos.to(x.device), sin.to(x.device)
|
||||
|
||||
if use_real_unbind_dim == -1:
|
||||
# Used for flux, cogvideox, hunyuan-dit
|
||||
x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, S, H, D//2]
|
||||
x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3)
|
||||
elif use_real_unbind_dim == -2:
|
||||
# Used for Stable Audio, OmniGen, CogView4 and Cosmos
|
||||
x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, S, H, D//2]
|
||||
x_rotated = torch.cat([-x_imag, x_real], dim=-1)
|
||||
else:
|
||||
raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.")
|
||||
|
||||
out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype)
|
||||
|
||||
return out
|
||||
else:
|
||||
x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2))
|
||||
freqs_cis = freqs_cis.unsqueeze(1)
|
||||
x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3)
|
||||
|
||||
return x_out.type_as(x)
|
||||
|
||||
|
||||
class QwenTimestepProjEmbeddings(nn.Module):
|
||||
def __init__(self, embedding_dim):
|
||||
super().__init__()
|
||||
|
||||
self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000)
|
||||
self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)
|
||||
|
||||
def forward(self, timestep, hidden_states):
|
||||
timesteps_proj = self.time_proj(timestep)
|
||||
timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype)) # (N, D)
|
||||
|
||||
conditioning = timesteps_emb
|
||||
|
||||
return conditioning
|
||||
|
||||
|
||||
class QwenEmbedRope(nn.Module):
|
||||
def __init__(self, theta: int, axes_dim: List[int], scale_rope=False):
|
||||
super().__init__()
|
||||
self.theta = theta
|
||||
self.axes_dim = axes_dim
|
||||
pos_index = torch.arange(4096)
|
||||
neg_index = torch.arange(4096).flip(0) * -1 - 1
|
||||
self.pos_freqs = torch.cat(
|
||||
[
|
||||
self.rope_params(pos_index, self.axes_dim[0], self.theta),
|
||||
self.rope_params(pos_index, self.axes_dim[1], self.theta),
|
||||
self.rope_params(pos_index, self.axes_dim[2], self.theta),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
self.neg_freqs = torch.cat(
|
||||
[
|
||||
self.rope_params(neg_index, self.axes_dim[0], self.theta),
|
||||
self.rope_params(neg_index, self.axes_dim[1], self.theta),
|
||||
self.rope_params(neg_index, self.axes_dim[2], self.theta),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
self.rope_cache = {}
|
||||
|
||||
# DO NOT USING REGISTER BUFFER HERE, IT WILL CAUSE COMPLEX NUMBERS LOSE ITS IMAGINARY PART
|
||||
self.scale_rope = scale_rope
|
||||
|
||||
def rope_params(self, index, dim, theta=10000):
|
||||
"""
|
||||
Args:
|
||||
index: [0, 1, 2, 3] 1D Tensor representing the position index of the token
|
||||
"""
|
||||
assert dim % 2 == 0
|
||||
freqs = torch.outer(index, 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim)))
|
||||
freqs = torch.polar(torch.ones_like(freqs), freqs)
|
||||
return freqs
|
||||
|
||||
def forward(self, video_fhw, txt_seq_lens, device):
|
||||
"""
|
||||
Args: video_fhw: [frame, height, width] a list of 3 integers representing the shape of the video Args:
|
||||
txt_length: [bs] a list of 1 integers representing the length of the text
|
||||
"""
|
||||
if self.pos_freqs.device != device:
|
||||
self.pos_freqs = self.pos_freqs.to(device)
|
||||
self.neg_freqs = self.neg_freqs.to(device)
|
||||
|
||||
if isinstance(video_fhw, list):
|
||||
video_fhw = video_fhw[0]
|
||||
if not isinstance(video_fhw, list):
|
||||
video_fhw = [video_fhw]
|
||||
|
||||
vid_freqs = []
|
||||
max_vid_index = 0
|
||||
for idx, fhw in enumerate(video_fhw):
|
||||
frame, height, width = fhw
|
||||
rope_key = f"{idx}_{height}_{width}"
|
||||
|
||||
if not torch.compiler.is_compiling():
|
||||
if rope_key not in self.rope_cache:
|
||||
self.rope_cache[rope_key] = self._compute_video_freqs(frame, height, width, idx)
|
||||
video_freq = self.rope_cache[rope_key]
|
||||
else:
|
||||
video_freq = self._compute_video_freqs(frame, height, width, idx)
|
||||
video_freq = video_freq.to(device)
|
||||
vid_freqs.append(video_freq)
|
||||
|
||||
if self.scale_rope:
|
||||
max_vid_index = max(height // 2, width // 2, max_vid_index)
|
||||
else:
|
||||
max_vid_index = max(height, width, max_vid_index)
|
||||
|
||||
max_len = max(txt_seq_lens)
|
||||
txt_freqs = self.pos_freqs[max_vid_index : max_vid_index + max_len, ...]
|
||||
vid_freqs = torch.cat(vid_freqs, dim=0)
|
||||
|
||||
return vid_freqs, txt_freqs
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _compute_video_freqs(self, frame, height, width, idx=0):
|
||||
seq_lens = frame * height * width
|
||||
freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1)
|
||||
freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1)
|
||||
|
||||
freqs_frame = freqs_pos[0][idx : idx + frame].view(frame, 1, 1, -1).expand(frame, height, width, -1)
|
||||
if self.scale_rope:
|
||||
freqs_height = torch.cat([freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], dim=0)
|
||||
freqs_height = freqs_height.view(1, height, 1, -1).expand(frame, height, width, -1)
|
||||
freqs_width = torch.cat([freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], dim=0)
|
||||
freqs_width = freqs_width.view(1, 1, width, -1).expand(frame, height, width, -1)
|
||||
else:
|
||||
freqs_height = freqs_pos[1][:height].view(1, height, 1, -1).expand(frame, height, width, -1)
|
||||
freqs_width = freqs_pos[2][:width].view(1, 1, width, -1).expand(frame, height, width, -1)
|
||||
|
||||
freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-1).reshape(seq_lens, -1)
|
||||
return freqs.clone().contiguous()
|
||||
|
||||
|
||||
class QwenDoubleStreamAttnProcessor2_0:
|
||||
"""
|
||||
Attention processor for Qwen double-stream architecture, matching DoubleStreamLayerMegatron logic. This processor
|
||||
implements joint attention computation where text and image streams are processed together.
|
||||
"""
|
||||
|
||||
_attention_backend = None
|
||||
|
||||
def __init__(self):
|
||||
if not hasattr(F, "scaled_dot_product_attention"):
|
||||
raise ImportError(
|
||||
"QwenDoubleStreamAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
attn: Attention,
|
||||
hidden_states: torch.FloatTensor, # Image stream
|
||||
encoder_hidden_states: torch.FloatTensor = None, # Text stream
|
||||
encoder_hidden_states_mask: torch.FloatTensor = None,
|
||||
attention_mask: Optional[torch.FloatTensor] = None,
|
||||
image_rotary_emb: Optional[torch.Tensor] = None,
|
||||
grag_scale = ((512,1.0,1.0),(4096,1.0,1.0)),
|
||||
) -> torch.FloatTensor:
|
||||
if encoder_hidden_states is None:
|
||||
raise ValueError("QwenDoubleStreamAttnProcessor2_0 requires encoder_hidden_states (text stream)")
|
||||
|
||||
seq_txt = encoder_hidden_states.shape[1]
|
||||
|
||||
# Compute QKV for image stream (sample projections)
|
||||
img_query = attn.to_q(hidden_states)
|
||||
img_key = attn.to_k(hidden_states)
|
||||
img_value = attn.to_v(hidden_states)
|
||||
|
||||
# Compute QKV for text stream (context projections)
|
||||
txt_query = attn.add_q_proj(encoder_hidden_states)
|
||||
txt_key = attn.add_k_proj(encoder_hidden_states)
|
||||
txt_value = attn.add_v_proj(encoder_hidden_states)
|
||||
|
||||
# Reshape for multi-head attention
|
||||
img_query = img_query.unflatten(-1, (attn.heads, -1))
|
||||
img_key = img_key.unflatten(-1, (attn.heads, -1))
|
||||
img_value = img_value.unflatten(-1, (attn.heads, -1))
|
||||
|
||||
txt_query = txt_query.unflatten(-1, (attn.heads, -1))
|
||||
txt_key = txt_key.unflatten(-1, (attn.heads, -1))
|
||||
txt_value = txt_value.unflatten(-1, (attn.heads, -1))
|
||||
|
||||
# Apply QK normalization
|
||||
if attn.norm_q is not None:
|
||||
img_query = attn.norm_q(img_query)
|
||||
if attn.norm_k is not None:
|
||||
img_key = attn.norm_k(img_key)
|
||||
if attn.norm_added_q is not None:
|
||||
txt_query = attn.norm_added_q(txt_query)
|
||||
if attn.norm_added_k is not None:
|
||||
txt_key = attn.norm_added_k(txt_key)
|
||||
|
||||
|
||||
# Apply RoPE
|
||||
if image_rotary_emb is not None:
|
||||
img_freqs, txt_freqs = image_rotary_emb
|
||||
img_query = apply_rotary_emb_qwen(img_query, img_freqs, use_real=False)
|
||||
img_key = apply_rotary_emb_qwen(img_key, img_freqs, use_real=False)
|
||||
txt_query = apply_rotary_emb_qwen(txt_query, txt_freqs, use_real=False)
|
||||
txt_key = apply_rotary_emb_qwen(txt_key, txt_freqs, use_real=False)
|
||||
|
||||
if grag_scale != (0,0) :
|
||||
txt_srag_scale , cond_srag_scale = grag_scale
|
||||
txt_len, txt_bias_scale, txt_delta_scale = txt_srag_scale
|
||||
img_len, img_bias_scale, img_delta_scale = cond_srag_scale
|
||||
|
||||
|
||||
txt_key_mean = txt_key[:,:txt_len,:,:].mean(dim=1)
|
||||
cond_key_mean = img_key[:,-1*img_len:,:,:].mean(dim=1)
|
||||
|
||||
|
||||
txt_key[:,:txt_len,:,:] = txt_bias_scale * txt_key_mean + (txt_key[:,:txt_len,:,:] - txt_key_mean) * txt_delta_scale
|
||||
img_key[:,-1*img_len:,:,:] = img_bias_scale * cond_key_mean + (img_key[:,-1*img_len:,:,:] - cond_key_mean) * img_delta_scale
|
||||
|
||||
# print(f"Implemented GRAG: txt_srag_scale:{txt_srag_scale}, cond_grag_scale: {cond_srag_scale}\n")
|
||||
|
||||
joint_query = torch.cat([txt_query, img_query], dim=1)
|
||||
joint_key = torch.cat([txt_key, img_key], dim=1)
|
||||
joint_value = torch.cat([txt_value, img_value], dim=1)
|
||||
# print(img_query.size())
|
||||
hook_out = {"query":joint_query.clone().detach(),
|
||||
"key":joint_key.clone().detach(),
|
||||
"value":joint_value.clone().detach()}
|
||||
|
||||
# Compute joint attention
|
||||
joint_hidden_states = dispatch_attention_fn(
|
||||
joint_query,
|
||||
joint_key,
|
||||
joint_value,
|
||||
attn_mask=attention_mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=False,
|
||||
backend=self._attention_backend,
|
||||
)
|
||||
|
||||
# Reshape back
|
||||
joint_hidden_states = joint_hidden_states.flatten(2, 3)
|
||||
joint_hidden_states = joint_hidden_states.to(joint_query.dtype)
|
||||
|
||||
# Split attention outputs back
|
||||
txt_attn_output = joint_hidden_states[:, :seq_txt, :] # Text part
|
||||
img_attn_output = joint_hidden_states[:, seq_txt:, :] # Image part
|
||||
|
||||
# Apply output projections
|
||||
img_attn_output = attn.to_out[0](img_attn_output)
|
||||
if len(attn.to_out) > 1:
|
||||
img_attn_output = attn.to_out[1](img_attn_output) # dropout
|
||||
|
||||
txt_attn_output = attn.to_add_out(txt_attn_output)
|
||||
|
||||
return (img_attn_output, txt_attn_output), hook_out
|
||||
|
||||
|
||||
@maybe_allow_in_graph
|
||||
class QwenImageTransformerBlock(nn.Module):
|
||||
def __init__(
|
||||
self, dim: int, num_attention_heads: int, attention_head_dim: int, qk_norm: str = "rms_norm", eps: float = 1e-6
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.dim = dim
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
|
||||
# Image processing modules
|
||||
self.img_mod = nn.Sequential(
|
||||
nn.SiLU(),
|
||||
nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2
|
||||
)
|
||||
self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.attn = Attention(
|
||||
query_dim=dim,
|
||||
cross_attention_dim=None, # Enable cross attention for joint computation
|
||||
added_kv_proj_dim=dim, # Enable added KV projections for text stream
|
||||
dim_head=attention_head_dim,
|
||||
heads=num_attention_heads,
|
||||
out_dim=dim,
|
||||
context_pre_only=False,
|
||||
bias=True,
|
||||
processor=QwenDoubleStreamAttnProcessor2_0(),
|
||||
qk_norm=qk_norm,
|
||||
eps=eps,
|
||||
)
|
||||
self.img_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.img_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
|
||||
|
||||
# Text processing modules
|
||||
self.txt_mod = nn.Sequential(
|
||||
nn.SiLU(),
|
||||
nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2
|
||||
)
|
||||
self.txt_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
# Text doesn't need separate attention - it's handled by img_attn joint computation
|
||||
self.txt_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
self.txt_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
|
||||
|
||||
def _modulate(self, x, mod_params):
|
||||
"""Apply modulation to input tensor"""
|
||||
shift, scale, gate = mod_params.chunk(3, dim=-1)
|
||||
return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1), gate.unsqueeze(1)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
encoder_hidden_states_mask: torch.Tensor,
|
||||
temb: torch.Tensor,
|
||||
image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
joint_attention_kwargs: Optional[Dict[str, Any]] = None,
|
||||
grag_scale = ((512,1.0,1.0),(512,1.0,1.0)),
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# Get modulation parameters for both streams
|
||||
img_mod_params = self.img_mod(temb) # [B, 6*dim]
|
||||
txt_mod_params = self.txt_mod(temb) # [B, 6*dim]
|
||||
|
||||
# Split modulation parameters for norm1 and norm2
|
||||
img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1) # Each [B, 3*dim]
|
||||
txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim]
|
||||
|
||||
# Process image stream - norm1 + modulation
|
||||
img_normed = self.img_norm1(hidden_states)
|
||||
img_modulated, img_gate1 = self._modulate(img_normed, img_mod1)
|
||||
|
||||
# Process text stream - norm1 + modulation
|
||||
txt_normed = self.txt_norm1(encoder_hidden_states)
|
||||
txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1)
|
||||
|
||||
# Use QwenAttnProcessor2_0 for joint attention computation
|
||||
# This directly implements the DoubleStreamLayerMegatron logic:
|
||||
# 1. Computes QKV for both streams
|
||||
# 2. Applies QK normalization and RoPE
|
||||
# 3. Concatenates and runs joint attention
|
||||
# 4. Splits results back to separate streams
|
||||
joint_attention_kwargs = joint_attention_kwargs or {}
|
||||
attn_output, hook_out = self.attn(
|
||||
hidden_states=img_modulated, # Image stream (will be processed as "sample")
|
||||
encoder_hidden_states=txt_modulated, # Text stream (will be processed as "context")
|
||||
encoder_hidden_states_mask=encoder_hidden_states_mask,
|
||||
image_rotary_emb=image_rotary_emb,
|
||||
grag_scale = grag_scale,
|
||||
**joint_attention_kwargs,
|
||||
)
|
||||
|
||||
hook_out = None
|
||||
del hook_out
|
||||
|
||||
# QwenAttnProcessor2_0 returns (img_output, txt_output) when encoder_hidden_states is provided
|
||||
img_attn_output, txt_attn_output = attn_output
|
||||
|
||||
# Apply attention gates and add residual (like in Megatron)
|
||||
hidden_states = hidden_states + img_gate1 * img_attn_output
|
||||
encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output
|
||||
|
||||
# Process image stream - norm2 + MLP
|
||||
img_normed2 = self.img_norm2(hidden_states)
|
||||
img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2)
|
||||
img_mlp_output = self.img_mlp(img_modulated2)
|
||||
hidden_states = hidden_states + img_gate2 * img_mlp_output
|
||||
|
||||
# Process text stream - norm2 + MLP
|
||||
txt_normed2 = self.txt_norm2(encoder_hidden_states)
|
||||
txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2)
|
||||
txt_mlp_output = self.txt_mlp(txt_modulated2)
|
||||
encoder_hidden_states = encoder_hidden_states + txt_gate2 * txt_mlp_output
|
||||
|
||||
# Clip to prevent overflow for fp16
|
||||
if encoder_hidden_states.dtype == torch.float16:
|
||||
encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504)
|
||||
if hidden_states.dtype == torch.float16:
|
||||
hidden_states = hidden_states.clip(-65504, 65504)
|
||||
|
||||
return encoder_hidden_states, hidden_states
|
||||
|
||||
|
||||
class QwenImageTransformer2DModel(
|
||||
ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin, CacheMixin, AttentionMixin
|
||||
):
|
||||
"""
|
||||
The Transformer model introduced in Qwen.
|
||||
|
||||
Args:
|
||||
patch_size (`int`, defaults to `2`):
|
||||
Patch size to turn the input data into small patches.
|
||||
in_channels (`int`, defaults to `64`):
|
||||
The number of channels in the input.
|
||||
out_channels (`int`, *optional*, defaults to `None`):
|
||||
The number of channels in the output. If not specified, it defaults to `in_channels`.
|
||||
num_layers (`int`, defaults to `60`):
|
||||
The number of layers of dual stream DiT blocks to use.
|
||||
attention_head_dim (`int`, defaults to `128`):
|
||||
The number of dimensions to use for each attention head.
|
||||
num_attention_heads (`int`, defaults to `24`):
|
||||
The number of attention heads to use.
|
||||
joint_attention_dim (`int`, defaults to `3584`):
|
||||
The number of dimensions to use for the joint attention (embedding/channel dimension of
|
||||
`encoder_hidden_states`).
|
||||
guidance_embeds (`bool`, defaults to `False`):
|
||||
Whether to use guidance embeddings for guidance-distilled variant of the model.
|
||||
axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`):
|
||||
The dimensions to use for the rotary positional embeddings.
|
||||
"""
|
||||
|
||||
_supports_gradient_checkpointing = True
|
||||
_no_split_modules = ["QwenImageTransformerBlock"]
|
||||
_skip_layerwise_casting_patterns = ["pos_embed", "norm"]
|
||||
_repeated_blocks = ["QwenImageTransformerBlock"]
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 2,
|
||||
in_channels: int = 64,
|
||||
out_channels: Optional[int] = 16,
|
||||
num_layers: int = 60,
|
||||
attention_head_dim: int = 128,
|
||||
num_attention_heads: int = 24,
|
||||
joint_attention_dim: int = 3584,
|
||||
guidance_embeds: bool = False, # TODO: this should probably be removed
|
||||
axes_dims_rope: Tuple[int, int, int] = (16, 56, 56),
|
||||
):
|
||||
super().__init__()
|
||||
self.out_channels = out_channels or in_channels
|
||||
self.inner_dim = num_attention_heads * attention_head_dim
|
||||
|
||||
self.pos_embed = QwenEmbedRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True)
|
||||
|
||||
self.time_text_embed = QwenTimestepProjEmbeddings(embedding_dim=self.inner_dim)
|
||||
|
||||
self.txt_norm = RMSNorm(joint_attention_dim, eps=1e-6)
|
||||
|
||||
self.img_in = nn.Linear(in_channels, self.inner_dim)
|
||||
self.txt_in = nn.Linear(joint_attention_dim, self.inner_dim)
|
||||
|
||||
self.transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
QwenImageTransformerBlock(
|
||||
dim=self.inner_dim,
|
||||
num_attention_heads=num_attention_heads,
|
||||
attention_head_dim=attention_head_dim,
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6)
|
||||
self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True)
|
||||
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor = None,
|
||||
encoder_hidden_states_mask: torch.Tensor = None,
|
||||
timestep: torch.LongTensor = None,
|
||||
img_shapes: Optional[List[Tuple[int, int, int]]] = None,
|
||||
txt_seq_lens: Optional[List[int]] = None,
|
||||
guidance: torch.Tensor = None, # TODO: this should probably be removed
|
||||
attention_kwargs: Optional[Dict[str, Any]] = None,
|
||||
controlnet_block_samples=None,
|
||||
return_dict: bool = True,
|
||||
grag_scale = [((512,1.0,1.0),(512,1.0,1.0))] * 60,
|
||||
) -> Union[torch.Tensor, Transformer2DModelOutput]:
|
||||
"""
|
||||
The [`QwenTransformer2DModel`] forward method.
|
||||
|
||||
Args:
|
||||
hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`):
|
||||
Input `hidden_states`.
|
||||
encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`):
|
||||
Conditional embeddings (embeddings computed from the input conditions such as prompts) to use.
|
||||
encoder_hidden_states_mask (`torch.Tensor` of shape `(batch_size, text_sequence_length)`):
|
||||
Mask of the input conditions.
|
||||
timestep ( `torch.LongTensor`):
|
||||
Used to indicate denoising step.
|
||||
attention_kwargs (`dict`, *optional*):
|
||||
A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
|
||||
`self.processor` in
|
||||
[diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
|
||||
return_dict (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain
|
||||
tuple.
|
||||
|
||||
Returns:
|
||||
If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a
|
||||
`tuple` where the first element is the sample tensor.
|
||||
"""
|
||||
if attention_kwargs is not None:
|
||||
attention_kwargs = attention_kwargs.copy()
|
||||
lora_scale = attention_kwargs.pop("scale", 1.0)
|
||||
else:
|
||||
lora_scale = 1.0
|
||||
|
||||
if USE_PEFT_BACKEND:
|
||||
# weight the lora layers by setting `lora_scale` for each PEFT layer
|
||||
scale_lora_layers(self, lora_scale)
|
||||
else:
|
||||
if attention_kwargs is not None and attention_kwargs.get("scale", None) is not None:
|
||||
logger.warning(
|
||||
"Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective."
|
||||
)
|
||||
|
||||
hidden_states = self.img_in(hidden_states)
|
||||
|
||||
timestep = timestep.to(hidden_states.dtype)
|
||||
encoder_hidden_states = self.txt_norm(encoder_hidden_states)
|
||||
encoder_hidden_states = self.txt_in(encoder_hidden_states)
|
||||
|
||||
if guidance is not None:
|
||||
guidance = guidance.to(hidden_states.dtype) * 1000
|
||||
|
||||
temb = (
|
||||
self.time_text_embed(timestep, hidden_states)
|
||||
if guidance is None
|
||||
else self.time_text_embed(timestep, guidance, hidden_states)
|
||||
)
|
||||
|
||||
image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
|
||||
|
||||
for index_block, block in enumerate(self.transformer_blocks):
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
encoder_hidden_states, hidden_states = self._gradient_checkpointing_func(
|
||||
block,
|
||||
hidden_states,
|
||||
encoder_hidden_states,
|
||||
encoder_hidden_states_mask,
|
||||
temb,
|
||||
image_rotary_emb,
|
||||
)
|
||||
|
||||
else:
|
||||
encoder_hidden_states, hidden_states = block(
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_hidden_states_mask=encoder_hidden_states_mask,
|
||||
temb=temb,
|
||||
image_rotary_emb=image_rotary_emb,
|
||||
joint_attention_kwargs=attention_kwargs,
|
||||
grag_scale = grag_scale.pop() ,
|
||||
)
|
||||
|
||||
# controlnet residual
|
||||
if controlnet_block_samples is not None:
|
||||
interval_control = len(self.transformer_blocks) / len(controlnet_block_samples)
|
||||
interval_control = int(np.ceil(interval_control))
|
||||
hidden_states = hidden_states + controlnet_block_samples[index_block // interval_control]
|
||||
|
||||
# Use only the image part (hidden_states) from the dual-stream blocks
|
||||
hidden_states = self.norm_out(hidden_states, temb)
|
||||
output = self.proj_out(hidden_states)
|
||||
|
||||
if USE_PEFT_BACKEND:
|
||||
# remove `lora_scale` from each PEFT layer
|
||||
unscale_lora_layers(self, lora_scale)
|
||||
|
||||
if not return_dict:
|
||||
return (output,)
|
||||
|
||||
return Transformer2DModelOutput(sample=output)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,564 +0,0 @@
|
||||
# Copyright 2025 Stability AI, Katherine Crowson and The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.utils import BaseOutput, is_scipy_available, logging
|
||||
from diffusers.schedulers.scheduling_utils import SchedulerMixin
|
||||
|
||||
if is_scipy_available():
|
||||
import scipy.stats
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput):
|
||||
"""
|
||||
Output class for the scheduler's `step` function output.
|
||||
|
||||
Args:
|
||||
prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
|
||||
Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the
|
||||
denoising loop.
|
||||
"""
|
||||
|
||||
prev_sample: torch.FloatTensor
|
||||
|
||||
|
||||
class FlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin):
|
||||
"""
|
||||
Euler scheduler.
|
||||
|
||||
This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic
|
||||
methods the library implements for all schedulers such as loading and saving.
|
||||
|
||||
Args:
|
||||
num_train_timesteps (`int`, defaults to 1000):
|
||||
The number of diffusion steps to train the model.
|
||||
shift (`float`, defaults to 1.0):
|
||||
The shift value for the timestep schedule.
|
||||
use_dynamic_shifting (`bool`, defaults to False):
|
||||
Whether to apply timestep shifting on-the-fly based on the image resolution.
|
||||
base_shift (`float`, defaults to 0.5):
|
||||
Value to stabilize image generation. Increasing `base_shift` reduces variation and image is more consistent
|
||||
with desired output.
|
||||
max_shift (`float`, defaults to 1.15):
|
||||
Value change allowed to latent vectors. Increasing `max_shift` encourages more variation and image may be
|
||||
more exaggerated or stylized.
|
||||
base_image_seq_len (`int`, defaults to 256):
|
||||
The base image sequence length.
|
||||
max_image_seq_len (`int`, defaults to 4096):
|
||||
The maximum image sequence length.
|
||||
invert_sigmas (`bool`, defaults to False):
|
||||
Whether to invert the sigmas.
|
||||
shift_terminal (`float`, defaults to None):
|
||||
The end value of the shifted timestep schedule.
|
||||
use_karras_sigmas (`bool`, defaults to False):
|
||||
Whether to use Karras sigmas for step sizes in the noise schedule during sampling.
|
||||
use_exponential_sigmas (`bool`, defaults to False):
|
||||
Whether to use exponential sigmas for step sizes in the noise schedule during sampling.
|
||||
use_beta_sigmas (`bool`, defaults to False):
|
||||
Whether to use beta sigmas for step sizes in the noise schedule during sampling.
|
||||
time_shift_type (`str`, defaults to "exponential"):
|
||||
The type of dynamic resolution-dependent timestep shifting to apply. Either "exponential" or "linear".
|
||||
stochastic_sampling (`bool`, defaults to False):
|
||||
Whether to use stochastic sampling.
|
||||
"""
|
||||
|
||||
_compatibles = []
|
||||
order = 1
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
num_train_timesteps: int = 1000,
|
||||
shift: float = 1.0,
|
||||
use_dynamic_shifting: bool = False,
|
||||
base_shift: Optional[float] = 0.5,
|
||||
max_shift: Optional[float] = 1.15,
|
||||
base_image_seq_len: Optional[int] = 256,
|
||||
max_image_seq_len: Optional[int] = 4096,
|
||||
invert_sigmas: bool = False,
|
||||
shift_terminal: Optional[float] = None,
|
||||
use_karras_sigmas: Optional[bool] = False,
|
||||
use_exponential_sigmas: Optional[bool] = False,
|
||||
use_beta_sigmas: Optional[bool] = False,
|
||||
time_shift_type: str = "exponential",
|
||||
stochastic_sampling: bool = False,
|
||||
):
|
||||
if self.config.use_beta_sigmas and not is_scipy_available():
|
||||
raise ImportError("Make sure to install scipy if you want to use beta sigmas.")
|
||||
if sum([self.config.use_beta_sigmas, self.config.use_exponential_sigmas, self.config.use_karras_sigmas]) > 1:
|
||||
raise ValueError(
|
||||
"Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, `config.use_karras_sigmas` can be used."
|
||||
)
|
||||
if time_shift_type not in {"exponential", "linear"}:
|
||||
raise ValueError("`time_shift_type` must either be 'exponential' or 'linear'.")
|
||||
|
||||
timesteps = np.linspace(1, num_train_timesteps, num_train_timesteps, dtype=np.float32)[::-1].copy()
|
||||
timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32)
|
||||
|
||||
sigmas = timesteps / num_train_timesteps
|
||||
if not use_dynamic_shifting:
|
||||
# when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution
|
||||
sigmas = shift * sigmas / (1 + (shift - 1) * sigmas)
|
||||
|
||||
self.timesteps = sigmas * num_train_timesteps
|
||||
|
||||
self._step_index = None
|
||||
self._begin_index = None
|
||||
|
||||
self._shift = shift
|
||||
|
||||
self.sigmas = sigmas.to("cpu") # to avoid too much CPU/GPU communication
|
||||
self.sigma_min = self.sigmas[-1].item()
|
||||
self.sigma_max = self.sigmas[0].item()
|
||||
|
||||
@property
|
||||
def shift(self):
|
||||
"""
|
||||
The value used for shifting.
|
||||
"""
|
||||
return self._shift
|
||||
|
||||
@property
|
||||
def step_index(self):
|
||||
"""
|
||||
The index counter for current timestep. It will increase 1 after each scheduler step.
|
||||
"""
|
||||
return self._step_index
|
||||
|
||||
@property
|
||||
def begin_index(self):
|
||||
"""
|
||||
The index for the first timestep. It should be set from pipeline with `set_begin_index` method.
|
||||
"""
|
||||
return self._begin_index
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index
|
||||
def set_begin_index(self, begin_index: int = 0):
|
||||
"""
|
||||
Sets the begin index for the scheduler. This function should be run from pipeline before the inference.
|
||||
|
||||
Args:
|
||||
begin_index (`int`):
|
||||
The begin index for the scheduler.
|
||||
"""
|
||||
self._begin_index = begin_index
|
||||
|
||||
def set_shift(self, shift: float):
|
||||
self._shift = shift
|
||||
|
||||
def scale_noise(
|
||||
self,
|
||||
sample: torch.FloatTensor,
|
||||
timestep: Union[float, torch.FloatTensor],
|
||||
noise: Optional[torch.FloatTensor] = None,
|
||||
) -> torch.FloatTensor:
|
||||
"""
|
||||
Forward process in flow-matching
|
||||
|
||||
Args:
|
||||
sample (`torch.FloatTensor`):
|
||||
The input sample.
|
||||
timestep (`int`, *optional*):
|
||||
The current timestep in the diffusion chain.
|
||||
|
||||
Returns:
|
||||
`torch.FloatTensor`:
|
||||
A scaled input sample.
|
||||
"""
|
||||
# Make sure sigmas and timesteps have the same device and dtype as original_samples
|
||||
sigmas = self.sigmas.to(device=sample.device, dtype=sample.dtype)
|
||||
|
||||
if sample.device.type == "mps" and torch.is_floating_point(timestep):
|
||||
# mps does not support float64
|
||||
schedule_timesteps = self.timesteps.to(sample.device, dtype=torch.float32)
|
||||
timestep = timestep.to(sample.device, dtype=torch.float32)
|
||||
else:
|
||||
schedule_timesteps = self.timesteps.to(sample.device)
|
||||
timestep = timestep.to(sample.device)
|
||||
|
||||
# self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
|
||||
if self.begin_index is None:
|
||||
step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timestep]
|
||||
elif self.step_index is not None:
|
||||
# add_noise is called after first denoising step (for inpainting)
|
||||
step_indices = [self.step_index] * timestep.shape[0]
|
||||
else:
|
||||
# add noise is called before first denoising step to create initial latent(img2img)
|
||||
step_indices = [self.begin_index] * timestep.shape[0]
|
||||
|
||||
sigma = sigmas[step_indices].flatten()
|
||||
while len(sigma.shape) < len(sample.shape):
|
||||
sigma = sigma.unsqueeze(-1)
|
||||
|
||||
sample = sigma * noise + (1.0 - sigma) * sample
|
||||
|
||||
return sample
|
||||
|
||||
def _sigma_to_t(self, sigma):
|
||||
return sigma * self.config.num_train_timesteps
|
||||
|
||||
def time_shift(self, mu: float, sigma: float, t: torch.Tensor):
|
||||
if self.config.time_shift_type == "exponential":
|
||||
return self._time_shift_exponential(mu, sigma, t)
|
||||
elif self.config.time_shift_type == "linear":
|
||||
return self._time_shift_linear(mu, sigma, t)
|
||||
|
||||
def stretch_shift_to_terminal(self, t: torch.Tensor) -> torch.Tensor:
|
||||
r"""
|
||||
Stretches and shifts the timestep schedule to ensure it terminates at the configured `shift_terminal` config
|
||||
value.
|
||||
|
||||
Reference:
|
||||
https://github.com/Lightricks/LTX-Video/blob/a01a171f8fe3d99dce2728d60a73fecf4d4238ae/ltx_video/schedulers/rf.py#L51
|
||||
|
||||
Args:
|
||||
t (`torch.Tensor`):
|
||||
A tensor of timesteps to be stretched and shifted.
|
||||
|
||||
Returns:
|
||||
`torch.Tensor`:
|
||||
A tensor of adjusted timesteps such that the final value equals `self.config.shift_terminal`.
|
||||
"""
|
||||
one_minus_z = 1 - t
|
||||
scale_factor = one_minus_z[-1] / (1 - self.config.shift_terminal)
|
||||
stretched_t = 1 - (one_minus_z / scale_factor)
|
||||
return stretched_t
|
||||
|
||||
def set_timesteps(
|
||||
self,
|
||||
num_inference_steps: Optional[int] = None,
|
||||
device: Union[str, torch.device] = None,
|
||||
sigmas: Optional[List[float]] = None,
|
||||
mu: Optional[float] = None,
|
||||
timesteps: Optional[List[float]] = None,
|
||||
):
|
||||
"""
|
||||
Sets the discrete timesteps used for the diffusion chain (to be run before inference).
|
||||
|
||||
Args:
|
||||
num_inference_steps (`int`, *optional*):
|
||||
The number of diffusion steps used when generating samples with a pre-trained model.
|
||||
device (`str` or `torch.device`, *optional*):
|
||||
The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
|
||||
sigmas (`List[float]`, *optional*):
|
||||
Custom values for sigmas to be used for each diffusion step. If `None`, the sigmas are computed
|
||||
automatically.
|
||||
mu (`float`, *optional*):
|
||||
Determines the amount of shifting applied to sigmas when performing resolution-dependent timestep
|
||||
shifting.
|
||||
timesteps (`List[float]`, *optional*):
|
||||
Custom values for timesteps to be used for each diffusion step. If `None`, the timesteps are computed
|
||||
automatically.
|
||||
"""
|
||||
if self.config.use_dynamic_shifting and mu is None:
|
||||
raise ValueError("`mu` must be passed when `use_dynamic_shifting` is set to be `True`")
|
||||
|
||||
if sigmas is not None and timesteps is not None:
|
||||
if len(sigmas) != len(timesteps):
|
||||
raise ValueError("`sigmas` and `timesteps` should have the same length")
|
||||
|
||||
if num_inference_steps is not None:
|
||||
if (sigmas is not None and len(sigmas) != num_inference_steps) or (
|
||||
timesteps is not None and len(timesteps) != num_inference_steps
|
||||
):
|
||||
raise ValueError(
|
||||
"`sigmas` and `timesteps` should have the same length as num_inference_steps, if `num_inference_steps` is provided"
|
||||
)
|
||||
else:
|
||||
num_inference_steps = len(sigmas) if sigmas is not None else len(timesteps)
|
||||
|
||||
self.num_inference_steps = num_inference_steps
|
||||
|
||||
# 1. Prepare default sigmas
|
||||
is_timesteps_provided = timesteps is not None
|
||||
|
||||
if is_timesteps_provided:
|
||||
timesteps = np.array(timesteps).astype(np.float32)
|
||||
|
||||
if sigmas is None:
|
||||
if timesteps is None:
|
||||
timesteps = np.linspace(
|
||||
self._sigma_to_t(self.sigma_max), self._sigma_to_t(self.sigma_min), num_inference_steps
|
||||
)
|
||||
sigmas = timesteps / self.config.num_train_timesteps
|
||||
else:
|
||||
sigmas = np.array(sigmas).astype(np.float32)
|
||||
num_inference_steps = len(sigmas)
|
||||
|
||||
# 2. Perform timestep shifting. Either no shifting is applied, or resolution-dependent shifting of
|
||||
# "exponential" or "linear" type is applied
|
||||
if self.config.use_dynamic_shifting:
|
||||
sigmas = self.time_shift(mu, 1.0, sigmas)
|
||||
else:
|
||||
sigmas = self.shift * sigmas / (1 + (self.shift - 1) * sigmas)
|
||||
|
||||
# 3. If required, stretch the sigmas schedule to terminate at the configured `shift_terminal` value
|
||||
if self.config.shift_terminal:
|
||||
sigmas = self.stretch_shift_to_terminal(sigmas)
|
||||
|
||||
# 4. If required, convert sigmas to one of karras, exponential, or beta sigma schedules
|
||||
if self.config.use_karras_sigmas:
|
||||
sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps)
|
||||
elif self.config.use_exponential_sigmas:
|
||||
sigmas = self._convert_to_exponential(in_sigmas=sigmas, num_inference_steps=num_inference_steps)
|
||||
elif self.config.use_beta_sigmas:
|
||||
sigmas = self._convert_to_beta(in_sigmas=sigmas, num_inference_steps=num_inference_steps)
|
||||
|
||||
# 5. Convert sigmas and timesteps to tensors and move to specified device
|
||||
sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32, device=device)
|
||||
if not is_timesteps_provided:
|
||||
timesteps = sigmas * self.config.num_train_timesteps
|
||||
else:
|
||||
timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32, device=device)
|
||||
|
||||
# 6. Append the terminal sigma value.
|
||||
# If a model requires inverted sigma schedule for denoising but timesteps without inversion, the
|
||||
# `invert_sigmas` flag can be set to `True`. This case is only required in Mochi
|
||||
if self.config.invert_sigmas:
|
||||
sigmas = 1.0 - sigmas
|
||||
timesteps = sigmas * self.config.num_train_timesteps
|
||||
sigmas = torch.cat([sigmas, torch.ones(1, device=sigmas.device)])
|
||||
else:
|
||||
sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)])
|
||||
|
||||
self.timesteps = timesteps
|
||||
self.sigmas = sigmas
|
||||
self._step_index = None
|
||||
self._begin_index = None
|
||||
|
||||
def index_for_timestep(self, timestep, schedule_timesteps=None):
|
||||
if schedule_timesteps is None:
|
||||
schedule_timesteps = self.timesteps
|
||||
|
||||
indices = (schedule_timesteps == timestep).nonzero()
|
||||
|
||||
# The sigma index that is taken for the **very** first `step`
|
||||
# is always the second index (or the last index if there is only 1)
|
||||
# This way we can ensure we don't accidentally skip a sigma in
|
||||
# case we start in the middle of the denoising schedule (e.g. for image-to-image)
|
||||
pos = 1 if len(indices) > 1 else 0
|
||||
|
||||
return indices[pos].item()
|
||||
|
||||
def _init_step_index(self, timestep):
|
||||
if self.begin_index is None:
|
||||
if isinstance(timestep, torch.Tensor):
|
||||
timestep = timestep.to(self.timesteps.device)
|
||||
self._step_index = self.index_for_timestep(timestep)
|
||||
else:
|
||||
self._step_index = self._begin_index
|
||||
|
||||
def step(
|
||||
self,
|
||||
model_output: torch.FloatTensor,
|
||||
timestep: Union[float, torch.FloatTensor],
|
||||
sample: torch.FloatTensor,
|
||||
s_churn: float = 0.0,
|
||||
s_tmin: float = 0.0,
|
||||
s_tmax: float = float("inf"),
|
||||
s_noise: float = 1.0,
|
||||
generator: Optional[torch.Generator] = None,
|
||||
per_token_timesteps: Optional[torch.Tensor] = None,
|
||||
return_dict: bool = True,
|
||||
) -> Union[FlowMatchEulerDiscreteSchedulerOutput, Tuple]:
|
||||
"""
|
||||
Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
|
||||
process from the learned model outputs (most often the predicted noise).
|
||||
|
||||
Args:
|
||||
model_output (`torch.FloatTensor`):
|
||||
The direct output from learned diffusion model.
|
||||
timestep (`float`):
|
||||
The current discrete timestep in the diffusion chain.
|
||||
sample (`torch.FloatTensor`):
|
||||
A current instance of a sample created by the diffusion process.
|
||||
s_churn (`float`):
|
||||
s_tmin (`float`):
|
||||
s_tmax (`float`):
|
||||
s_noise (`float`, defaults to 1.0):
|
||||
Scaling factor for noise added to the sample.
|
||||
generator (`torch.Generator`, *optional*):
|
||||
A random number generator.
|
||||
per_token_timesteps (`torch.Tensor`, *optional*):
|
||||
The timesteps for each token in the sample.
|
||||
return_dict (`bool`):
|
||||
Whether or not to return a
|
||||
[`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] or tuple.
|
||||
|
||||
Returns:
|
||||
[`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] or `tuple`:
|
||||
If return_dict is `True`,
|
||||
[`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] is returned,
|
||||
otherwise a tuple is returned where the first element is the sample tensor.
|
||||
"""
|
||||
|
||||
if (
|
||||
isinstance(timestep, int)
|
||||
or isinstance(timestep, torch.IntTensor)
|
||||
or isinstance(timestep, torch.LongTensor)
|
||||
):
|
||||
raise ValueError(
|
||||
(
|
||||
"Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
|
||||
" `FlowMatchEulerDiscreteScheduler.step()` is not supported. Make sure to pass"
|
||||
" one of the `scheduler.timesteps` as a timestep."
|
||||
),
|
||||
)
|
||||
|
||||
if self.step_index is None:
|
||||
self._init_step_index(timestep)
|
||||
|
||||
# Upcast to avoid precision issues when computing prev_sample
|
||||
sample = sample.to(torch.float32)
|
||||
|
||||
if per_token_timesteps is not None:
|
||||
per_token_sigmas = per_token_timesteps / self.config.num_train_timesteps
|
||||
|
||||
sigmas = self.sigmas[:, None, None]
|
||||
lower_mask = sigmas < per_token_sigmas[None] - 1e-6
|
||||
lower_sigmas = lower_mask * sigmas
|
||||
lower_sigmas, _ = lower_sigmas.max(dim=0)
|
||||
|
||||
current_sigma = per_token_sigmas[..., None]
|
||||
next_sigma = lower_sigmas[..., None]
|
||||
dt = current_sigma - next_sigma
|
||||
else:
|
||||
sigma_idx = self.step_index
|
||||
sigma = self.sigmas[sigma_idx]
|
||||
sigma_next = self.sigmas[sigma_idx + 1]
|
||||
|
||||
current_sigma = sigma
|
||||
next_sigma = sigma_next
|
||||
dt = sigma_next - sigma
|
||||
|
||||
if self.config.stochastic_sampling:
|
||||
x0 = sample - current_sigma * model_output
|
||||
noise = torch.randn_like(sample)
|
||||
prev_sample = (1.0 - next_sigma) * x0 + next_sigma * noise
|
||||
|
||||
else:
|
||||
prev_sample = sample + dt * model_output
|
||||
|
||||
pred_original_sample = sample + (self.sigmas[-1] - sigma) * model_output
|
||||
|
||||
# upon completion increase step index by one
|
||||
self._step_index += 1
|
||||
if per_token_timesteps is None:
|
||||
# Cast sample back to model compatible dtype
|
||||
prev_sample = prev_sample.to(model_output.dtype)
|
||||
pred_original_sample = pred_original_sample.to(model_output.dtype)
|
||||
|
||||
if not return_dict:
|
||||
return (prev_sample,pred_original_sample)
|
||||
|
||||
return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample)
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras
|
||||
def _convert_to_karras(self, in_sigmas: torch.Tensor, num_inference_steps) -> torch.Tensor:
|
||||
"""Constructs the noise schedule of Karras et al. (2022)."""
|
||||
|
||||
# Hack to make sure that other schedulers which copy this function don't break
|
||||
# TODO: Add this logic to the other schedulers
|
||||
if hasattr(self.config, "sigma_min"):
|
||||
sigma_min = self.config.sigma_min
|
||||
else:
|
||||
sigma_min = None
|
||||
|
||||
if hasattr(self.config, "sigma_max"):
|
||||
sigma_max = self.config.sigma_max
|
||||
else:
|
||||
sigma_max = None
|
||||
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
|
||||
rho = 7.0 # 7.0 is the value used in the paper
|
||||
ramp = np.linspace(0, 1, num_inference_steps)
|
||||
min_inv_rho = sigma_min ** (1 / rho)
|
||||
max_inv_rho = sigma_max ** (1 / rho)
|
||||
sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
|
||||
return sigmas
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential
|
||||
def _convert_to_exponential(self, in_sigmas: torch.Tensor, num_inference_steps: int) -> torch.Tensor:
|
||||
"""Constructs an exponential noise schedule."""
|
||||
|
||||
# Hack to make sure that other schedulers which copy this function don't break
|
||||
# TODO: Add this logic to the other schedulers
|
||||
if hasattr(self.config, "sigma_min"):
|
||||
sigma_min = self.config.sigma_min
|
||||
else:
|
||||
sigma_min = None
|
||||
|
||||
if hasattr(self.config, "sigma_max"):
|
||||
sigma_max = self.config.sigma_max
|
||||
else:
|
||||
sigma_max = None
|
||||
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
|
||||
sigmas = np.exp(np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps))
|
||||
return sigmas
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta
|
||||
def _convert_to_beta(
|
||||
self, in_sigmas: torch.Tensor, num_inference_steps: int, alpha: float = 0.6, beta: float = 0.6
|
||||
) -> torch.Tensor:
|
||||
"""From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)"""
|
||||
|
||||
# Hack to make sure that other schedulers which copy this function don't break
|
||||
# TODO: Add this logic to the other schedulers
|
||||
if hasattr(self.config, "sigma_min"):
|
||||
sigma_min = self.config.sigma_min
|
||||
else:
|
||||
sigma_min = None
|
||||
|
||||
if hasattr(self.config, "sigma_max"):
|
||||
sigma_max = self.config.sigma_max
|
||||
else:
|
||||
sigma_max = None
|
||||
|
||||
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
||||
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
||||
|
||||
sigmas = np.array(
|
||||
[
|
||||
sigma_min + (ppf * (sigma_max - sigma_min))
|
||||
for ppf in [
|
||||
scipy.stats.beta.ppf(timestep, alpha, beta)
|
||||
for timestep in 1 - np.linspace(0, 1, num_inference_steps)
|
||||
]
|
||||
]
|
||||
)
|
||||
return sigmas
|
||||
|
||||
def _time_shift_exponential(self, mu, sigma, t):
|
||||
return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma)
|
||||
|
||||
def _time_shift_linear(self, mu, sigma, t):
|
||||
return mu / (mu + (1 / t - 1) ** sigma)
|
||||
|
||||
def __len__(self):
|
||||
return self.config.num_train_timesteps
|
||||
@@ -1,396 +0,0 @@
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import json
|
||||
import cv2
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from safetensors.torch import load_file as load_sft
|
||||
import gc
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, List, Optional, Union
|
||||
from accelerate.big_modeling import dispatch_model
|
||||
import logging
|
||||
from accelerate.utils import (
|
||||
find_tied_parameters,
|
||||
get_balanced_memory,
|
||||
infer_auto_device_map,
|
||||
retie_parameters,
|
||||
)
|
||||
from accelerate.utils.offload import offload_weight, save_offload_index
|
||||
from accelerate.utils.modeling import (
|
||||
check_tied_parameters_in_config,
|
||||
check_tied_parameters_on_same_device,
|
||||
set_module_tensor_to_device,
|
||||
load_offloaded_weights
|
||||
)
|
||||
from accelerate.utils.constants import SAFE_WEIGHTS_NAME, WEIGHTS_NAME
|
||||
import shutil
|
||||
import tempfile
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def load_checkpoint_and_dispatch_(
|
||||
model: nn.Module,
|
||||
sd: Union[str, os.PathLike],
|
||||
device_map: Optional[Union[str, Dict[str, Union[int, str, torch.device]]]] = None,
|
||||
max_memory: Optional[Dict[Union[int, str], Union[int, str]]] = None,
|
||||
no_split_module_classes: Optional[List[str]] = None,
|
||||
offload_folder: Optional[Union[str, os.PathLike]] = None,
|
||||
offload_buffers: bool = False,
|
||||
dtype: Optional[Union[str, torch.dtype]] = None,
|
||||
offload_state_dict: Optional[bool] = None,
|
||||
skip_keys: Optional[Union[str, List[str]]] = None,
|
||||
preload_module_classes: Optional[List[str]] = None,
|
||||
force_hooks: bool = False,
|
||||
strict: bool = False,
|
||||
):
|
||||
"""
|
||||
Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are
|
||||
loaded and adds the various hooks that will make this model run properly (even if split across devices).
|
||||
|
||||
Args:
|
||||
model (`torch.nn.Module`): The model in which we want to load a checkpoint.
|
||||
sd (`dict`):
|
||||
# checkpoint (`str` or `os.PathLike`):
|
||||
# The folder checkpoint to load. It can be:
|
||||
# - a path to a file containing a whole model state dict
|
||||
# - a path to a `.json` file containing the index to a sharded checkpoint
|
||||
# - a path to a folder containing a unique `.index.json` file and the shards of a checkpoint.
|
||||
device_map (`Dict[str, Union[int, str, torch.device]]`, *optional*):
|
||||
A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer
|
||||
name, once a given module name is inside, every submodule of it will be sent to the same device.
|
||||
|
||||
To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For more
|
||||
information about each option see [here](../concept_guides/big_model_inference#designing-a-device-map).
|
||||
Defaults to None, which means [`dispatch_model`] will not be called.
|
||||
max_memory (`Dict`, *optional*):
|
||||
A dictionary device identifier to maximum memory. Will default to the maximum memory available for each GPU
|
||||
and the available CPU RAM if unset.
|
||||
no_split_module_classes (`List[str]`, *optional*):
|
||||
A list of layer class names that should never be split across device (for instance any layer that has a
|
||||
residual connection).
|
||||
offload_folder (`str` or `os.PathLike`, *optional*):
|
||||
If the `device_map` contains any value `"disk"`, the folder where we will offload weights.
|
||||
offload_buffers (`bool`, *optional*, defaults to `False`):
|
||||
In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as
|
||||
well as the parameters.
|
||||
dtype (`str` or `torch.dtype`, *optional*):
|
||||
If provided, the weights will be converted to that type when loaded.
|
||||
offload_state_dict (`bool`, *optional*):
|
||||
If `True`, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if
|
||||
the weight of the CPU state dict + the biggest shard does not fit. Will default to `True` if the device map
|
||||
picked contains `"disk"` values.
|
||||
skip_keys (`str` or `List[str]`, *optional*):
|
||||
A list of keys to ignore when moving inputs or outputs between devices.
|
||||
preload_module_classes (`List[str]`, *optional*):
|
||||
A list of classes whose instances should load all their weights (even in the submodules) at the beginning
|
||||
of the forward. This should only be used for classes that have submodules which are registered but not
|
||||
called directly during the forward, for instance if a `dense` linear layer is registered, but at forward,
|
||||
`dense.weight` and `dense.bias` are used in some operations instead of calling `dense` directly.
|
||||
force_hooks (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not to force device hooks to be attached to the model even if all layers are dispatched to a
|
||||
single device.
|
||||
strict (`bool`, *optional*, defaults to `False`):
|
||||
Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model's
|
||||
state_dict.
|
||||
|
||||
Example:
|
||||
|
||||
```python
|
||||
>>> from accelerate import init_empty_weights, load_checkpoint_and_dispatch
|
||||
>>> from huggingface_hub import hf_hub_download
|
||||
>>> from transformers import AutoConfig, AutoModelForCausalLM
|
||||
|
||||
>>> # Download the Weights
|
||||
>>> checkpoint = "EleutherAI/gpt-j-6B"
|
||||
>>> weights_location = hf_hub_download(checkpoint, "pytorch_model.bin")
|
||||
|
||||
>>> # Create a model and initialize it with empty weights
|
||||
>>> config = AutoConfig.from_pretrained(checkpoint)
|
||||
>>> with init_empty_weights():
|
||||
... model = AutoModelForCausalLM.from_config(config)
|
||||
|
||||
>>> # Load the checkpoint and dispatch it to the right devices
|
||||
>>> model = load_checkpoint_and_dispatch(
|
||||
... model, weights_location, device_map="auto", no_split_module_classes=["GPTJBlock"]
|
||||
... )
|
||||
```
|
||||
"""
|
||||
if isinstance(device_map, str) and device_map not in ["auto", "balanced", "balanced_low_0", "sequential"]:
|
||||
raise ValueError(
|
||||
"If passing a string for `device_map`, please choose 'auto', 'balanced', 'balanced_low_0' or "
|
||||
"'sequential'."
|
||||
)
|
||||
if isinstance(device_map, str):
|
||||
if device_map != "sequential":
|
||||
max_memory = get_balanced_memory(
|
||||
model,
|
||||
max_memory=max_memory,
|
||||
no_split_module_classes=no_split_module_classes,
|
||||
dtype=dtype,
|
||||
low_zero=(device_map == "balanced_low_0"),
|
||||
)
|
||||
device_map = infer_auto_device_map(
|
||||
model,
|
||||
max_memory=max_memory,
|
||||
no_split_module_classes=no_split_module_classes,
|
||||
dtype=dtype,
|
||||
offload_buffers=offload_buffers,
|
||||
)
|
||||
if offload_state_dict is None and device_map is not None and "disk" in device_map.values():
|
||||
offload_state_dict = True
|
||||
load_checkpoint_in_model_(
|
||||
model,
|
||||
sd,
|
||||
device_map=device_map,
|
||||
offload_folder=offload_folder,
|
||||
dtype=dtype,
|
||||
offload_state_dict=offload_state_dict,
|
||||
offload_buffers=offload_buffers,
|
||||
strict=strict,
|
||||
)
|
||||
if device_map is None:
|
||||
return model
|
||||
return dispatch_model(
|
||||
model,
|
||||
device_map=device_map,
|
||||
offload_dir=offload_folder,
|
||||
offload_buffers=offload_buffers,
|
||||
skip_keys=skip_keys,
|
||||
preload_module_classes=preload_module_classes,
|
||||
force_hooks=force_hooks,
|
||||
)
|
||||
|
||||
|
||||
|
||||
def load_checkpoint_in_model_(
|
||||
model: nn.Module,
|
||||
sd,
|
||||
device_map: Optional[Dict[str, Union[int, str, torch.device]]] = None,
|
||||
offload_folder: Optional[Union[str, os.PathLike]] = None,
|
||||
dtype: Optional[Union[str, torch.dtype]] = None,
|
||||
offload_state_dict: bool = False,
|
||||
offload_buffers: bool = False,
|
||||
keep_in_fp32_modules: List[str] = None,
|
||||
offload_8bit_bnb: bool = False,
|
||||
strict: bool = False,
|
||||
):
|
||||
"""
|
||||
Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are
|
||||
loaded.
|
||||
|
||||
<Tip warning={true}>
|
||||
|
||||
Once loaded across devices, you still need to call [`dispatch_model`] on your model to make it able to run. To
|
||||
group the checkpoint loading and dispatch in one single call, use [`load_checkpoint_and_dispatch`].
|
||||
|
||||
</Tip>
|
||||
|
||||
Args:
|
||||
model (`torch.nn.Module`):
|
||||
The model in which we want to load a checkpoint.
|
||||
sd,
|
||||
# checkpoint (`str` or `os.PathLike`):
|
||||
# The folder checkpoint to load. It can be:
|
||||
# - a path to a file containing a whole model state dict
|
||||
# - a path to a `.json` file containing the index to a sharded checkpoint
|
||||
# - a path to a folder containing a unique `.index.json` file and the shards of a checkpoint.
|
||||
# - a path to a folder containing a unique pytorch_model.bin or a model.safetensors file.
|
||||
device_map (`Dict[str, Union[int, str, torch.device]]`, *optional*):
|
||||
A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer
|
||||
name, once a given module name is inside, every submodule of it will be sent to the same device.
|
||||
offload_folder (`str` or `os.PathLike`, *optional*):
|
||||
If the `device_map` contains any value `"disk"`, the folder where we will offload weights.
|
||||
dtype (`str` or `torch.dtype`, *optional*):
|
||||
If provided, the weights will be converted to that type when loaded.
|
||||
offload_state_dict (`bool`, *optional*, defaults to `False`):
|
||||
If `True`, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if
|
||||
the weight of the CPU state dict + the biggest shard does not fit.
|
||||
offload_buffers (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not to include the buffers in the weights offloaded to disk.
|
||||
keep_in_fp32_modules(`List[str]`, *optional*):
|
||||
A list of the modules that we keep in `torch.float32` dtype.
|
||||
offload_8bit_bnb (`bool`, *optional*):
|
||||
Whether or not to enable offload of 8-bit modules on cpu/disk.
|
||||
strict (`bool`, *optional*, defaults to `False`):
|
||||
Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model's
|
||||
state_dict.
|
||||
|
||||
"""
|
||||
if offload_8bit_bnb:
|
||||
from accelerate.utils.bnb import quantize_and_offload_8bit
|
||||
|
||||
tied_params = find_tied_parameters(model)
|
||||
|
||||
if check_tied_parameters_in_config(model) and len(tied_params) == 0:
|
||||
logger.warn(
|
||||
"The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function."
|
||||
)
|
||||
if device_map is not None:
|
||||
check_tied_parameters_on_same_device(tied_params, device_map)
|
||||
|
||||
if offload_folder is None and device_map is not None and "disk" in device_map.values():
|
||||
raise ValueError(
|
||||
"At least one of the model submodule will be offloaded to disk, please pass along an `offload_folder`."
|
||||
)
|
||||
elif offload_folder is not None and device_map is not None and "disk" in device_map.values():
|
||||
os.makedirs(offload_folder, exist_ok=True)
|
||||
|
||||
if isinstance(dtype, str):
|
||||
# We accept "torch.float16" or just "float16"
|
||||
dtype = dtype.replace("torch.", "")
|
||||
dtype = getattr(torch, dtype)
|
||||
|
||||
#checkpoint_files = None
|
||||
#index_filename = None
|
||||
# if os.path.isfile(checkpoint):
|
||||
# if str(checkpoint).endswith(".json"):
|
||||
# index_filename = checkpoint
|
||||
# else:
|
||||
# checkpoint_files = [checkpoint]
|
||||
# elif os.path.isdir(checkpoint):
|
||||
# # check if the whole state dict is present
|
||||
# potential_state_bin = [f for f in os.listdir(checkpoint) if f == WEIGHTS_NAME]
|
||||
# potential_state_safetensor = [f for f in os.listdir(checkpoint) if f == SAFE_WEIGHTS_NAME]
|
||||
# if len(potential_state_bin) == 1:
|
||||
# checkpoint_files = [os.path.join(checkpoint, potential_state_bin[0])]
|
||||
# elif len(potential_state_safetensor) == 1:
|
||||
# checkpoint_files = [os.path.join(checkpoint, potential_state_safetensor[0])]
|
||||
# else:
|
||||
# # otherwise check for sharded checkpoints
|
||||
# potential_index = [f for f in os.listdir(checkpoint) if f.endswith(".index.json")]
|
||||
# if len(potential_index) == 0:
|
||||
# raise ValueError(
|
||||
# f"{checkpoint} is not a folder containing a `.index.json` file or a {WEIGHTS_NAME} or a {SAFE_WEIGHTS_NAME} file"
|
||||
# )
|
||||
# elif len(potential_index) == 1:
|
||||
# index_filename = os.path.join(checkpoint, potential_index[0])
|
||||
# else:
|
||||
# raise ValueError(
|
||||
# f"{checkpoint} containing more than one `.index.json` file, delete the irrelevant ones."
|
||||
# )
|
||||
# else:
|
||||
# raise ValueError(
|
||||
# "`checkpoint` should be the path to a file containing a whole state dict, or the index of a sharded "
|
||||
# f"checkpoint, or a folder containing a sharded checkpoint or the whole state dict, but got {checkpoint}."
|
||||
# )
|
||||
|
||||
# if index_filename is not None:
|
||||
# checkpoint_folder = os.path.split(index_filename)[0]
|
||||
# with open(index_filename) as f:
|
||||
# index = json.loads(f.read())
|
||||
|
||||
# if "weight_map" in index:
|
||||
# index = index["weight_map"]
|
||||
# checkpoint_files = sorted(list(set(index.values())))
|
||||
# checkpoint_files = [os.path.join(checkpoint_folder, f) for f in checkpoint_files]
|
||||
|
||||
# Logic for missing/unexepected keys goes here.
|
||||
|
||||
offload_index = {}
|
||||
if offload_state_dict:
|
||||
state_dict_folder = tempfile.mkdtemp()
|
||||
state_dict_index = {}
|
||||
|
||||
unexpected_keys = set()
|
||||
model_keys = set(model.state_dict().keys())
|
||||
buffer_names = [name for name, _ in model.named_buffers()]
|
||||
|
||||
for loaded_checkpoint in [sd]:
|
||||
#loaded_checkpoint = load_state_dict(checkpoint_file, device_map=device_map)
|
||||
if device_map is None:
|
||||
model.load_state_dict(loaded_checkpoint, strict=strict)
|
||||
unexpected_keys.update(set(loaded_checkpoint.keys()) - model_keys)
|
||||
else:
|
||||
for param_name, param in loaded_checkpoint.items():
|
||||
# skip SCB parameter (for 8-bit serialization)
|
||||
if "SCB" in param_name:
|
||||
continue
|
||||
|
||||
if param_name not in model_keys:
|
||||
unexpected_keys.add(param_name)
|
||||
if not strict:
|
||||
continue # Skip loading this parameter.
|
||||
|
||||
module_name = param_name
|
||||
|
||||
while len(module_name) > 0 and module_name not in device_map:
|
||||
module_name = ".".join(module_name.split(".")[:-1])
|
||||
if module_name == "" and "" not in device_map:
|
||||
# TODO: group all errors and raise at the end.
|
||||
raise ValueError(f"{param_name} doesn't have any device set.")
|
||||
param_device = device_map[module_name]
|
||||
new_dtype = dtype
|
||||
if dtype is not None and torch.is_floating_point(param):
|
||||
if keep_in_fp32_modules is not None and dtype == torch.float16:
|
||||
proceed = False
|
||||
for key in keep_in_fp32_modules:
|
||||
if ((key in param_name) and (key + "." in param_name)) or key == param_name:
|
||||
proceed = True
|
||||
break
|
||||
if proceed:
|
||||
new_dtype = torch.float32
|
||||
|
||||
if "weight" in param_name and param_name.replace("weight", "SCB") in loaded_checkpoint.keys():
|
||||
if param.dtype == torch.int8:
|
||||
fp16_statistics = loaded_checkpoint[param_name.replace("weight", "SCB")]
|
||||
else:
|
||||
fp16_statistics = None
|
||||
|
||||
if param_device == "disk":
|
||||
if offload_buffers or param_name not in buffer_names:
|
||||
if new_dtype is None:
|
||||
new_dtype = param.dtype
|
||||
if offload_8bit_bnb:
|
||||
quantize_and_offload_8bit(
|
||||
model, param, param_name, new_dtype, offload_folder, offload_index, fp16_statistics
|
||||
)
|
||||
continue
|
||||
else:
|
||||
set_module_tensor_to_device(model, param_name, "meta", dtype=new_dtype)
|
||||
offload_weight(param, param_name, offload_folder, index=offload_index)
|
||||
elif param_device == "cpu" and offload_state_dict:
|
||||
if new_dtype is None:
|
||||
new_dtype = param.dtype
|
||||
if offload_8bit_bnb:
|
||||
quantize_and_offload_8bit(
|
||||
model, param, param_name, new_dtype, state_dict_folder, state_dict_index, fp16_statistics
|
||||
)
|
||||
else:
|
||||
set_module_tensor_to_device(model, param_name, "meta", dtype=new_dtype)
|
||||
offload_weight(param, param_name, state_dict_folder, index=state_dict_index)
|
||||
else:
|
||||
set_module_tensor_to_device(
|
||||
model,
|
||||
param_name,
|
||||
param_device,
|
||||
value=param,
|
||||
dtype=new_dtype,
|
||||
fp16_statistics=fp16_statistics,
|
||||
)
|
||||
|
||||
# Force Python to clean up.
|
||||
del loaded_checkpoint
|
||||
gc.collect()
|
||||
|
||||
if not strict and len(unexpected_keys) > 0:
|
||||
logger.warning(
|
||||
f"Some weights of the model checkpoint were not used when"
|
||||
f" initializing {model.__class__.__name__}: {unexpected_keys}. This may or may not be an issue - make sure that the checkpoint does not have unnecessary parameters, or that the model definition correctly corresponds to the checkpoint."
|
||||
)
|
||||
|
||||
save_offload_index(offload_index, offload_folder)
|
||||
|
||||
# Load back offloaded state dict on CPU
|
||||
if offload_state_dict:
|
||||
load_offloaded_weights(model, state_dict_index, state_dict_folder)
|
||||
shutil.rmtree(state_dict_folder)
|
||||
|
||||
retie_parameters(model, tied_params)
|
||||
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
import math
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from typing import Dict, List, Literal
|
||||
import matplotlib.cm as cm
|
||||
import matplotlib as mpl
|
||||
from mpl_toolkits.mplot3d import Axes3D
|
||||
import os
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from base64 import b64encode
|
||||
from PIL import Image
|
||||
import random
|
||||
import io
|
||||
from io import BytesIO
|
||||
|
||||
def seed_everything(seed: int = 42, deterministic: bool = False):
|
||||
|
||||
random.seed(seed)
|
||||
os.environ["PYTHONHASHSEED"] = str(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
torch.cuda.manual_seed(seed)
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
if deterministic:
|
||||
torch.backends.cudnn.deterministic = True
|
||||
torch.backends.cudnn.benchmark = False
|
||||
else:
|
||||
torch.backends.cudnn.deterministic = False
|
||||
torch.backends.cudnn.benchmark = True
|
||||
|
||||
print(f"✅ Random seed set to {seed}, deterministic={deterministic}")
|
||||
@@ -1,118 +0,0 @@
|
||||
import os
|
||||
import torch
|
||||
import sys
|
||||
from termcolor import colored
|
||||
from diffusers import QwenImageEditPipeline,QuantoConfig
|
||||
from .hacked_models.scheduler import FlowMatchEulerDiscreteScheduler
|
||||
from .hacked_models.pipeline import QwenImageEditPipeline
|
||||
from .hacked_models.models import QwenImageTransformer2DModel
|
||||
from .hacked_models.pipeline_plus import QwenImageEditPlusPipeline
|
||||
import sys
|
||||
from .hacked_models.utils import *
|
||||
from contextlib import contextmanager
|
||||
from accelerate import init_empty_weights
|
||||
import sys
|
||||
from .util import load_checkpoint_and_dispatch_
|
||||
from accelerate.big_modeling import dispatch_model,get_balanced_memory,infer_auto_device_map
|
||||
@contextmanager
|
||||
def temp_patch_module_attr(module_name: str, attr_name: str, new_obj):
|
||||
mod = sys.modules.get(module_name)
|
||||
if mod is None:
|
||||
yield
|
||||
return
|
||||
had = hasattr(mod, attr_name)
|
||||
orig = getattr(mod, attr_name, None)
|
||||
setattr(mod, attr_name, new_obj)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if had:
|
||||
setattr(mod, attr_name, orig)
|
||||
else:
|
||||
try:
|
||||
delattr(mod, attr_name)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def load_model(gguf_path,unet_path,node_path):
|
||||
plus_mode=False
|
||||
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit/scheduler"),torch_dtype=torch.bfloat16,)
|
||||
if gguf_path is not None:
|
||||
if "2509" in gguf_path.lower() or "plus" in gguf_path.lower():
|
||||
plus_mode=True
|
||||
from diffusers import GGUFQuantizationConfig
|
||||
with temp_patch_module_attr("diffusers", "QwenImageTransformer2DModel", QwenImageTransformer2DModel):
|
||||
transformer = QwenImageTransformer2DModel.from_single_file(
|
||||
gguf_path,
|
||||
config=os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit/transformer"),
|
||||
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
if "2509" in unet_path.lower() or "plus" in unet_path.lower():
|
||||
plus_mode=True
|
||||
|
||||
try:
|
||||
print("loading from single file safetensors")
|
||||
with temp_patch_module_attr("diffusers", "QwenImageTransformer2DModel", QwenImageTransformer2DModel):
|
||||
transformer = QwenImageTransformer2DModel.from_single_file(unet_path,config=os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit/transformer"),torch_dtype=torch.bfloat16,)
|
||||
except:
|
||||
print("loading from single file failed, try to load from single file from dict")
|
||||
from safetensors.torch import load_file
|
||||
sd=load_file(unet_path)
|
||||
sd=replace_key(sd)
|
||||
with init_empty_weights():
|
||||
with temp_patch_module_attr("diffusers", "QwenImageTransformer2DModel", QwenImageTransformer2DModel):
|
||||
unet_config = QwenImageTransformer2DModel.load_config(os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit/transformer/config.json"))
|
||||
transformer = QwenImageTransformer2DModel.from_config(unet_config).to(torch.bfloat16)
|
||||
model_state_dict = transformer.state_dict()
|
||||
expected_keys = set(model_state_dict.keys())
|
||||
del transformer
|
||||
filtered_sd = {k: v for k, v in sd.items() if k in expected_keys}
|
||||
# 检查是否有缺失的键
|
||||
missing_keys = expected_keys - set(sd.keys())
|
||||
if missing_keys:
|
||||
print(f"Warning: Missing keys in checkpoint: {missing_keys}")
|
||||
del sd
|
||||
with temp_patch_module_attr("diffusers", "QwenImageTransformer2DModel", QwenImageTransformer2DModel):
|
||||
transformer = QwenImageTransformer2DModel.from_single_file(filtered_sd,config=os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit/transformer"),torch_dtype=torch.bfloat16,)
|
||||
del filtered_sd
|
||||
|
||||
if plus_mode:
|
||||
pipeline = QwenImageEditPlusPipeline.from_pretrained(os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit"), scheduler = scheduler,vae=None,text_encoder=None,transformer=transformer,torch_dtype=torch.bfloat16,)
|
||||
else:
|
||||
pipeline = QwenImageEditPipeline.from_pretrained(os.path.join(node_path, "Qwen_Edit_GRAG/Qwen-Image-Edit"), scheduler = scheduler,vae=None,text_encoder=None,transformer=transformer,torch_dtype=torch.bfloat16,)
|
||||
|
||||
return pipeline
|
||||
|
||||
def replace_key(t_state_dict):
|
||||
return {k.replace("model.diffusion_model.", "", 1): v for k, v in t_state_dict.items()}
|
||||
|
||||
def inference(pipeline,positive,negative,num_inference_steps,seed,true_cfg_scale,cond_b,cond_delta):
|
||||
|
||||
seed_everything(seed)
|
||||
pipeline.set_progress_bar_config(disable=None)
|
||||
|
||||
inputs = {
|
||||
"image":None,
|
||||
"prompt": None,
|
||||
"generator": torch.manual_seed(seed),
|
||||
"true_cfg_scale": true_cfg_scale,
|
||||
"negative_prompt": None,
|
||||
"num_inference_steps": num_inference_steps,
|
||||
"prompt_embeds": positive[0][0],
|
||||
"negative_prompt_embeds": negative[0][0],
|
||||
"image_latents":positive[0][1].get("reference_latents",None) ,
|
||||
"return_dict": False,
|
||||
"grag_scale":[((512,1.0,1.0),(4096,cond_b,cond_delta))]*60,
|
||||
}
|
||||
|
||||
with torch.inference_mode():
|
||||
output = pipeline(**inputs)
|
||||
image,x0_images,saved_outputs = output
|
||||
|
||||
#image[0].save(os.path.join(out_path,f"{args.image_path.split('/')[-1]}_cond_b-{args.cond_b}_cond_delta-{args.cond_delta}.jpg"))
|
||||
return image,x0_images
|
||||
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
accelerate
|
||||
diffusers
|
||||
gradio
|
||||
ipykernel
|
||||
jaraco.collections
|
||||
matplotlib
|
||||
pathlib
|
||||
pickleshare
|
||||
pip-chill
|
||||
termcolor
|
||||
tomli
|
||||
torch
|
||||
torchvision
|
||||
transformers
|
||||
huggingface_hub
|
||||
@@ -1,96 +0,0 @@
|
||||
import os
|
||||
from PIL import Image
|
||||
import torch
|
||||
import sys
|
||||
import json
|
||||
import tqdm
|
||||
from pathlib import Path
|
||||
from termcolor import colored
|
||||
from diffusers import QwenImageEditPipeline
|
||||
from hacked_models.scheduler import FlowMatchEulerDiscreteScheduler
|
||||
from hacked_models.pipeline import QwenImageEditPipeline
|
||||
from hacked_models.models import QwenImageTransformer2DModel
|
||||
import sys
|
||||
import json
|
||||
import tqdm
|
||||
import argparse
|
||||
from hacked_models.utils import *
|
||||
seed_everything(42)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument("--model_path", type=str, default="Qwen/Qwen-Image-Edit")
|
||||
parser.add_argument("--data_root", type=str, required=True)
|
||||
parser.add_argument("--out_path", type=str, required=True)
|
||||
parser.add_argument("--name", type=str, required=True)
|
||||
parser.add_argument("--cond_b", type=float, required=True)
|
||||
parser.add_argument("--cond_delta", type=float, required=True)
|
||||
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
|
||||
os.path.join(args.model_path, "scheduler"),
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
transformer = QwenImageTransformer2DModel.from_pretrained(
|
||||
os.path.join(args.model_path, "transformer"),
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
pipeline = QwenImageEditPipeline.from_pretrained(args.model_path, torch_dtype=torch.bfloat16,
|
||||
scheduler = scheduler,
|
||||
transformer=transformer,
|
||||
)
|
||||
print("pipeline loaded")
|
||||
pipeline.to(torch.bfloat16)
|
||||
pipeline.to("cuda")
|
||||
pipeline.set_progress_bar_config(disable=None)
|
||||
|
||||
DATA_ROOT = args.data_root
|
||||
data_root = '/'.join(DATA_ROOT.split('/')[:-1])
|
||||
out_path = os.path.join(args.out_path, DATA_ROOT.split('/')[-2],args.name)
|
||||
os.makedirs(out_path,exist_ok=True)
|
||||
|
||||
with open (DATA_ROOT , 'r') as f:
|
||||
test_pairs = json.load(f)
|
||||
print(colored(out_path,color = "green"))
|
||||
|
||||
|
||||
|
||||
for name , pair in tqdm.tqdm(list(test_pairs.items())):
|
||||
|
||||
sample_name = name
|
||||
img_folder, img_name = '/'.join(pair["image_path"].split('/')[:-1]) , pair["image_path"].split('/')[-1]
|
||||
# original_prompt = pair["original_prompt"]
|
||||
# editing_prompt = pair["editing_prompt"]
|
||||
editing_instruction = pair["editing_instruction"]
|
||||
|
||||
input_image = Image.open(os.path.join(data_root,img_folder,img_name)).convert('RGB')
|
||||
os.makedirs(os.path.join(out_path,img_folder),exist_ok=True)
|
||||
os.makedirs(os.path.join(out_path,img_folder),exist_ok=True)
|
||||
if os.path.exists(os.path.join(out_path,img_folder,img_name)):
|
||||
print('Already generated.')
|
||||
continue
|
||||
grid = []
|
||||
prompt = editing_instruction
|
||||
inputs = {
|
||||
"image": input_image,
|
||||
"prompt": prompt,
|
||||
"generator": torch.manual_seed(42),
|
||||
"true_cfg_scale": 4.0,
|
||||
"negative_prompt": " ",
|
||||
"num_inference_steps": 24,
|
||||
"return_dict": False,
|
||||
"grag_scale":[((512,1.0,1.0),(4096,args.cond_b,args.cond_delta))]*60,
|
||||
}
|
||||
|
||||
with torch.inference_mode():
|
||||
output = pipeline(**inputs)
|
||||
image,x0_images,saved_outputs = output
|
||||
|
||||
image[0].save(os.path.join(out_path,img_folder,img_name))
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import math
|
||||
import numpy as np
|
||||
import torch
|
||||
import os
|
||||
from diffusers.hooks import apply_group_offloading
|
||||
import folder_paths
|
||||
from typing_extensions import override
|
||||
from comfy_api.latest import ComfyExtension, io
|
||||
import nodes
|
||||
import comfy.model_management as mm
|
||||
from .model_loader_utils import tensor2list,nomarl_upscale,get_emb_data
|
||||
from .Qwen_Edit_GRAG.inference import load_model,inference
|
||||
from .Qwen_Edit_GRAG.hacked_models.scheduler import FlowMatchEulerDiscreteScheduler
|
||||
from .Qwen_Edit_GRAG.hacked_models.pipeline_plus import QwenImageEditPlusPipeline
|
||||
import node_helpers
|
||||
MAX_SEED = np.iinfo(np.int32).max
|
||||
node_cr_path = os.path.dirname(os.path.abspath(__file__))
|
||||
device = torch.device(
|
||||
"cuda:0") if torch.cuda.is_available() else torch.device(
|
||||
"mps") if torch.backends.mps.is_available() else torch.device(
|
||||
"cpu")
|
||||
|
||||
|
||||
weigths_gguf_current_path = os.path.join(folder_paths.models_dir, "gguf")
|
||||
if not os.path.exists(weigths_gguf_current_path):
|
||||
os.makedirs(weigths_gguf_current_path)
|
||||
|
||||
folder_paths.add_model_folder_path("gguf", weigths_gguf_current_path) # gguf dir
|
||||
|
||||
|
||||
class Qwen_Edit_GRAG_SM_Model(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
|
||||
return io.Schema(
|
||||
node_id="Qwen_Edit_GRAG_SM_Model",
|
||||
display_name="Qwen_Edit_GRAG_SM_Model",
|
||||
category="Qwen_Edit_GRAG",
|
||||
inputs=[
|
||||
io.Combo.Input("dit",options= ["none"] + folder_paths.get_filename_list("diffusion_models") ),
|
||||
io.Combo.Input("gguf",options= ["none"] + folder_paths.get_filename_list("gguf") ),
|
||||
],
|
||||
outputs=[
|
||||
io.Custom("Qwen_Edit_GRAG_SM_Model").Output(display_name="model"),
|
||||
],
|
||||
)
|
||||
@classmethod
|
||||
def execute(cls, dit,gguf) -> io.NodeOutput:
|
||||
dit_path=folder_paths.get_full_path("diffusion_models", dit) if dit != "none" else None
|
||||
gguf_path=folder_paths.get_full_path("gguf", gguf) if gguf != "none" else None
|
||||
pipeline = load_model(gguf_path,dit_path,node_cr_path)
|
||||
return io.NodeOutput(pipeline)
|
||||
|
||||
class Qwen_Edit_GRAG_SM_Encode(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="Qwen_Edit_GRAG_SM_Encode",
|
||||
display_name="Qwen_Edit_GRAG_SM_Encode",
|
||||
category="Qwen_Edit_GRAG",
|
||||
inputs=[
|
||||
io.Clip.Input("clip"),
|
||||
io.Vae.Input("vae"),
|
||||
io.Image.Input("image"),
|
||||
io.Int.Input("width", default=1024, min=256, max=nodes.MAX_RESOLUTION,step=16,display_mode=io.NumberDisplay.number),
|
||||
io.Int.Input("height", default=1024, min=256, max=nodes.MAX_RESOLUTION,step=16,display_mode=io.NumberDisplay.number),
|
||||
io.String.Input("pos_text", multiline=True,default=",best"),
|
||||
io.String.Input("neg_text", multiline=True,default="bad anatomy, bad hands, missing fingers, extra fingers,three hands, three legs, bad arms, missing legs, missing arms, poorly drawn face, bad face, fused face, cloned face, three crus, fused feet, fused thigh, extra crus, ugly fingers, horn,amputation, disconnected limbs"),
|
||||
|
||||
],
|
||||
outputs=[
|
||||
io.Conditioning.Output(display_name="positive"),
|
||||
io.Conditioning.Output(display_name="negative"),
|
||||
],
|
||||
)
|
||||
@classmethod
|
||||
def execute(cls, clip, vae,image,width,height,pos_text,neg_text,) -> io.NodeOutput:
|
||||
|
||||
tensor_list=tensor2list(image,width,height)
|
||||
#pli_image=nomarl_upscale(image,width,height) if isinstance(image,torch.Tensor) else None
|
||||
|
||||
postive,ref_latents=get_emb_data(clip,vae,pos_text,tensor_list,plus=True if len(tensor_list) > 1 else False)
|
||||
if len(tensor_list) > 1:
|
||||
negative,_=get_emb_data(clip,vae,neg_text,tensor_list,plus=True)
|
||||
else:
|
||||
negative,_=get_emb_data(clip,vae,neg_text,tensor_list,ng=True,img=tensor_list[0])
|
||||
postive=node_helpers.conditioning_set_values(postive, {"reference_latents": ref_latents})
|
||||
|
||||
# gc cf model
|
||||
cf_models=mm.loaded_models()
|
||||
try:
|
||||
for pipe in cf_models:
|
||||
pipe.unpatch_model(device_to=torch.device("cpu"))
|
||||
print(f"Unpatching models.{pipe}")
|
||||
except: pass
|
||||
mm.soft_empty_cache()
|
||||
torch.cuda.empty_cache()
|
||||
max_gpu_memory = torch.cuda.max_memory_allocated()
|
||||
print(f"After Max GPU memory allocated: {max_gpu_memory / 1000 ** 3:.2f} GB")
|
||||
|
||||
return io.NodeOutput(postive,negative)
|
||||
|
||||
class Qwen_Edit_GRAG_SM_KSampler(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="Qwen_Edit_GRAG_SM_KSampler",
|
||||
display_name="Qwen_Edit_GRAG_SM_KSampler",
|
||||
category="Qwen_Edit_GRAG",
|
||||
inputs=[
|
||||
io.Custom("Qwen_Edit_GRAG_SM_Model").Input("model"),
|
||||
io.Conditioning.Input("positive"),
|
||||
io.Conditioning.Input("negative"),
|
||||
io.Combo.Input("lora",options= ["none"] + folder_paths.get_filename_list("loras")),
|
||||
io.Int.Input("steps", default=24, min=1, max=1024,step=1,display_mode=io.NumberDisplay.number),
|
||||
io.Float.Input("guidance_scale", default=4.0, min=0, max=20,step=0.01,display_mode=io.NumberDisplay.number),
|
||||
io.Int.Input("seed", default=0, min=0, max=MAX_SEED,display_mode=io.NumberDisplay.number),
|
||||
io.Float.Input("cond_b", default=1.0, min=0.01, max=10.0,step=0.01,display_mode=io.NumberDisplay.number),
|
||||
io.Float.Input("cond_delta", default=1.10, min=0.01, max=10.0,step=0.01,display_mode=io.NumberDisplay.number),
|
||||
io.Int.Input("block_num", default=10, min=1, max=MAX_SEED,display_mode=io.NumberDisplay.number),
|
||||
],
|
||||
outputs=[
|
||||
io.Latent.Output(display_name="latent"),
|
||||
io.Latent.Output(display_name="latents"),
|
||||
],
|
||||
)
|
||||
@classmethod
|
||||
def execute(cls, model,positive,negative,lora,steps,guidance_scale,seed,cond_b,cond_delta,block_num,) -> io.NodeOutput:
|
||||
|
||||
if isinstance(model,QwenImageEditPlusPipeline) and not isinstance(positive[0][1].get("reference_latents"),list):
|
||||
raise "###### when use edit 2509 need two image input. 使用2509plus模型时.encoder需要输入2张图片. ######"
|
||||
|
||||
adapter_path=folder_paths.get_full_path("loras", lora) if lora != "none" else None
|
||||
if adapter_path is not None:
|
||||
model.load_lora_weights(adapter_path,weight_name= os.path.basename(adapter_path))
|
||||
scheduler_config = {
|
||||
"base_image_seq_len": 256,
|
||||
"base_shift": math.log(3), # We use shift=3 in distillation
|
||||
"invert_sigmas": False,
|
||||
"max_image_seq_len": 8192,
|
||||
"max_shift": math.log(3), # We use shift=3 in distillation
|
||||
"num_train_timesteps": 1000,
|
||||
"shift": 1.0,
|
||||
"shift_terminal": None, # set shift_terminal to None
|
||||
"stochastic_sampling": False,
|
||||
"time_shift_type": "exponential",
|
||||
"use_beta_sigmas": False,
|
||||
"use_dynamic_shifting": True,
|
||||
"use_exponential_sigmas": False,
|
||||
"use_karras_sigmas": False,
|
||||
}
|
||||
model.scheduler = FlowMatchEulerDiscreteScheduler.from_config(scheduler_config)
|
||||
|
||||
|
||||
# apply offloading
|
||||
apply_group_offloading(model.transformer, onload_device=torch.device("cuda"), offload_type="block_level", num_blocks_per_group=block_num)
|
||||
# infer
|
||||
lat,lats=inference(model,positive,negative,steps,seed,guidance_scale,cond_b,cond_delta)
|
||||
lats=torch.cat(lats,dim=0)
|
||||
out_put={"samples":lat}
|
||||
out_puts={"samples":lats}
|
||||
return io.NodeOutput(out_put,out_puts)
|
||||
|
||||
|
||||
|
||||
from aiohttp import web
|
||||
from server import PromptServer
|
||||
@PromptServer.instance.routes.get("/Qwen_Edit_GRAG_SM_Extension")
|
||||
async def get_hello(request):
|
||||
return web.json_response("Qwen_Edit_GRAG_SM_Extension")
|
||||
|
||||
class Qwen_Edit_GRAG_SM_Extension(ComfyExtension):
|
||||
@override
|
||||
async def get_node_list(self) -> list[type[io.ComfyNode]]:
|
||||
return [
|
||||
Qwen_Edit_GRAG_SM_Model,
|
||||
Qwen_Edit_GRAG_SM_Encode,
|
||||
Qwen_Edit_GRAG_SM_KSampler,
|
||||
]
|
||||
async def comfy_entrypoint() -> Qwen_Edit_GRAG_SM_Extension: # ComfyUI calls this to load your extension and its nodes.
|
||||
return Qwen_Edit_GRAG_SM_Extension()
|
||||
|
||||
|
||||
|
||||
@@ -1,47 +1,55 @@
|
||||
# ComfyUI_GRAG_Image_Editing
|
||||
[GRAG-Image-Editing](https://github.com/little-misfit/GRAG-Image-Editing) : Group-Relative Attention Guidance for Image Editing,you can try it in comfyUI
|
||||
|
||||
[GRAG-Image-Editing](https://github.com/little-misfit/GRAG-Image-Editing) : Group-Relative Attention Guidance for Image Editing, on the **native comfy DiT stack**.
|
||||
|
||||
|
||||
# Update
|
||||
* fix safetensor load error ,if use safetensor need Ram >64 / 使用单体safetensor 加载时，尤其是comfy的量化版本，因为diffuser要二次处理，对内存占用较大，所以推荐用fp16的模型或者gguf，此插件对量化的safetensor模型内存优化支持一般；
|
||||
* add 2509 support / 增加2509模型的支持，注意要输入2张以上的图片
|
||||
* 改为comfyUI核心封装/change to comfyUI core Wrapper
|
||||
|
||||
|
||||
1.Installation
|
||||
# Installation
|
||||
-----
|
||||
In the ./ComfyUI/custom_nodes directory, run the following:
|
||||
```
|
||||
git clone https://github.com/smthemex/ComfyUI_GRAG_Image_Editing
|
||||
```
|
||||
2.requirements
|
||||
----
|
||||
* 不装也行，没什么需求，diffuser版本高点
|
||||
```
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
3.Model
|
||||
----
|
||||
|
||||
* gguf or transformer [smthem/Qwen-Image-GGUF](https://huggingface.co/smthem/Qwen-Image-GGUF/tree/main) or other or comfy-org optional/随便哪个gguf或者comfyUI官方的transformer
|
||||
* comfyUI normal： qwen-image vae and qwen_2.5_vl_7b
|
||||
# Models
|
||||
----
|
||||
|
||||
* comfyUI normal：dit, qwen-image vae and qwen_2.5_vl_7b
|
||||
* lora, lightx2v 8step lora# 千问edit加速
|
||||
```
|
||||
├── ComfyUI/models/gguf # or transformer
|
||||
| ├── Qwen-Image-BF16.gguf # or Q8
|
||||
|
||||
├── ComfyUI/models/diffusion_models # or gguf
|
||||
| ├── Qwen-Image-BF16..safetensors # or e4m3fn
|
||||
| ├── qwen_image_edit_2511_fp8mixed.safetensors
|
||||
├── ComfyUI/models/vae
|
||||
| ├─Qwen-Image.safetensors # rename it 换个名字
|
||||
| ├─Qwen-Image.safetensors
|
||||
├── ComfyUI/models/clip
|
||||
| ├──qwen_2.5_vl_7b_fp8_scaled.safetensors
|
||||
├── ComfyUI/models/loras
|
||||
| ├──Qwen-Image-Edit-Lightning-8steps-V1.0-bf16.safetensors
|
||||
|
||||
| ├──Qwen-Image-Edit-2511-Lightning-4steps-V1.0-bf16.safetensors
|
||||
```
|
||||
|
||||
|
||||
# Example
|
||||
|
||||
|
||||
### GRAGPatch parameters
|
||||
|
||||
- `cond_delta` — the GRAG strength. **< 1 frees the output from the reference (stronger edit);
|
||||
> 1 locks onto the reference image.** Upstream recommended sweep: `0.8 ~ 1.7`, step `0.01`,
|
||||
default `1.10`. Verified on 2511 (cfg 4.0, 24 steps): `1.70 → corr 0.995 with the input`,
|
||||
`1.10 → 0.997`, `0.90 → 0.928`, `0.80 → 0.711`.
|
||||
- `cond_b` — bias (lambda) scale; upstream keeps it at `1.0` everywhere. Leave it alone.
|
||||
- `img_len` — how many **trailing** image tokens form the reference group. comfy's
|
||||
`TextEncodeQwenImageEdit` resizes the reference to 1024×1024 → 4096 tokens, which is the
|
||||
default and matches upstream's hardcoded value. Use `img_len = (ref_w/16) * (ref_h/16)`
|
||||
summed over references when you deviate from that (e.g. `TextEncodeQwenImageEditPlus` with
|
||||
multiple images, or a custom non-resizing encoder). `0` = the whole image block.
|
||||
- `txt_delta` / `txt_b` — same reweighting applied to the text group; upstream no-ops (1.0).
|
||||
- `apply_from` / `apply_to` — sigma window (denoising progress 0→1). Defaults cover all steps,
|
||||
matching upstream.
|
||||
- `verbose` — print the effective config when the graph runs.
|
||||
|
||||
# Citation
|
||||
```
|
||||
@misc{zhang2025grouprelativeattentionguidance,
|
||||
|
||||
+17
-1
@@ -1,2 +1,18 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
from .Qwen_Edit_GRAG_node import *
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from comfy_api.latest import ComfyExtension, io
|
||||
|
||||
from .grag_nodes import GRAGPatch
|
||||
|
||||
|
||||
class GRAG_Extension(ComfyExtension):
|
||||
@override
|
||||
async def get_node_list(self) -> list[type[io.ComfyNode]]:
|
||||
return [GRAGPatch]
|
||||
|
||||
|
||||
async def comfy_entrypoint() -> GRAG_Extension:
|
||||
return GRAG_Extension()
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 158 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 535 KiB After Width: | Height: | Size: 472 KiB |
@@ -1,591 +0,0 @@
|
||||
{
|
||||
"id": "362dbf13-d0cd-4bbf-959e-5cc9a16a87a3",
|
||||
"revision": 0,
|
||||
"last_node_id": 13,
|
||||
"last_link_id": 19,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "Qwen_Edit_GRAG_SM_Model",
|
||||
"pos": [
|
||||
3278.5674596178287,
|
||||
944.5528134829161
|
||||
],
|
||||
"size": [
|
||||
292.09921875,
|
||||
82
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "Qwen_Edit_GRAG_SM_Model",
|
||||
"links": [
|
||||
1
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Qwen_Edit_GRAG_SM_Model"
|
||||
},
|
||||
"widgets_values": [
|
||||
"none",
|
||||
"qwen-image-edit-q6_k.gguf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "Qwen_Edit_GRAG_SM_KSampler",
|
||||
"pos": [
|
||||
3726.856011210847,
|
||||
926.1772496981262
|
||||
],
|
||||
"size": [
|
||||
315.437109375,
|
||||
266
|
||||
],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "Qwen_Edit_GRAG_SM_Model",
|
||||
"link": 1
|
||||
},
|
||||
{
|
||||
"name": "positive",
|
||||
"type": "CONDITIONING",
|
||||
"link": 17
|
||||
},
|
||||
{
|
||||
"name": "negative",
|
||||
"type": "CONDITIONING",
|
||||
"link": 18
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "latent",
|
||||
"type": "LATENT",
|
||||
"links": [
|
||||
2
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "latents",
|
||||
"type": "LATENT",
|
||||
"links": [
|
||||
10
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Qwen_Edit_GRAG_SM_KSampler"
|
||||
},
|
||||
"widgets_values": [
|
||||
"wanqwen\\Qwen-Image-Edit-Lightning-8steps-V1.0-bf16.safetensors",
|
||||
8,
|
||||
1,
|
||||
1487597530,
|
||||
"randomize",
|
||||
1,
|
||||
1.1,
|
||||
5
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "VAEDecode",
|
||||
"pos": [
|
||||
4144.2988951101315,
|
||||
962.5264307927719
|
||||
],
|
||||
"size": [
|
||||
140,
|
||||
46
|
||||
],
|
||||
"flags": {},
|
||||
"order": 9,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "samples",
|
||||
"type": "LATENT",
|
||||
"link": 10
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"link": 11
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
14
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VAEDecode"
|
||||
},
|
||||
"widgets_values": []
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "VAELoader",
|
||||
"pos": [
|
||||
2869.471522685459,
|
||||
1278.2584160799624
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
58
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "VAE",
|
||||
"type": "VAE",
|
||||
"links": [
|
||||
9,
|
||||
11,
|
||||
16
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VAELoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"qwen_image_vae.safetensors"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "VAEDecode",
|
||||
"pos": [
|
||||
3684.926771486949,
|
||||
1246.9549577170362
|
||||
],
|
||||
"size": [
|
||||
140,
|
||||
46
|
||||
],
|
||||
"flags": {},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "samples",
|
||||
"type": "LATENT",
|
||||
"link": 2
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"link": 9
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
8
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VAEDecode"
|
||||
},
|
||||
"widgets_values": []
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "SaveImage",
|
||||
"pos": [
|
||||
3299.9605300016055,
|
||||
1400.9197714942366
|
||||
],
|
||||
"size": [
|
||||
310,
|
||||
318.88885498046875
|
||||
],
|
||||
"flags": {},
|
||||
"order": 10,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 8
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"ComfyUI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "CreateVideo",
|
||||
"pos": [
|
||||
4145.723220394224,
|
||||
1061.6173074240744
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
78
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "VIDEO",
|
||||
"type": "VIDEO",
|
||||
"links": [
|
||||
13
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CreateVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
8
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "SaveVideo",
|
||||
"pos": [
|
||||
4096.440452283425,
|
||||
1324.849694481293
|
||||
],
|
||||
"size": [
|
||||
544.9999999999995,
|
||||
392.4444519042968
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"link": 13
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"video/ComfyUI",
|
||||
"auto",
|
||||
"auto"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "SaveImage",
|
||||
"pos": [
|
||||
3661.743279875754,
|
||||
1383.1931513738266
|
||||
],
|
||||
"size": [
|
||||
404.242453835227,
|
||||
356.888916015625
|
||||
],
|
||||
"flags": {},
|
||||
"order": 11,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 14
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"ComfyUI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "CLIPLoader",
|
||||
"pos": [
|
||||
2869.471415873932,
|
||||
1118.3694451326971
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
106
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "CLIP",
|
||||
"type": "CLIP",
|
||||
"links": [
|
||||
15
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CLIPLoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"qwen_2.5_vl_7b_fp8_scaled.safetensors",
|
||||
"qwen_image",
|
||||
"default"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "Qwen_Edit_GRAG_SM_Encode",
|
||||
"pos": [
|
||||
3190.286015407823,
|
||||
1089.3169496209416
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
230
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": 15
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"link": 16
|
||||
},
|
||||
{
|
||||
"name": "image",
|
||||
"type": "IMAGE",
|
||||
"link": 19
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "positive",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
17
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "negative",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
18
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Qwen_Edit_GRAG_SM_Encode"
|
||||
},
|
||||
"widgets_values": [
|
||||
1024,
|
||||
1024,
|
||||
"Turn the real birds to origami birds.",
|
||||
"bad anatomy, bad hands, missing fingers, extra fingers,three hands, three legs, bad arms, missing legs, missing arms, poorly drawn face, bad face, fused face, cloned face, three crus, fused feet, fused thigh, extra crus, ugly fingers, horn,amputation, disconnected limbs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "LoadImage",
|
||||
"pos": [
|
||||
3006.058478970681,
|
||||
1388.6276844685826
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
314
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
19
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadImage"
|
||||
},
|
||||
"widgets_values": [
|
||||
"000000000020.jpg",
|
||||
"image"
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
2,
|
||||
0,
|
||||
"Qwen_Edit_GRAG_SM_Model"
|
||||
],
|
||||
[
|
||||
2,
|
||||
2,
|
||||
0,
|
||||
3,
|
||||
0,
|
||||
"LATENT"
|
||||
],
|
||||
[
|
||||
8,
|
||||
3,
|
||||
0,
|
||||
8,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
9,
|
||||
6,
|
||||
0,
|
||||
3,
|
||||
1,
|
||||
"VAE"
|
||||
],
|
||||
[
|
||||
10,
|
||||
2,
|
||||
1,
|
||||
9,
|
||||
0,
|
||||
"LATENT"
|
||||
],
|
||||
[
|
||||
11,
|
||||
6,
|
||||
0,
|
||||
9,
|
||||
1,
|
||||
"VAE"
|
||||
],
|
||||
[
|
||||
13,
|
||||
10,
|
||||
0,
|
||||
11,
|
||||
0,
|
||||
"VIDEO"
|
||||
],
|
||||
[
|
||||
14,
|
||||
9,
|
||||
0,
|
||||
12,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
15,
|
||||
5,
|
||||
0,
|
||||
13,
|
||||
0,
|
||||
"CLIP"
|
||||
],
|
||||
[
|
||||
16,
|
||||
6,
|
||||
0,
|
||||
13,
|
||||
1,
|
||||
"VAE"
|
||||
],
|
||||
[
|
||||
17,
|
||||
13,
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
"CONDITIONING"
|
||||
],
|
||||
[
|
||||
18,
|
||||
13,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
"CONDITIONING"
|
||||
],
|
||||
[
|
||||
19,
|
||||
7,
|
||||
0,
|
||||
13,
|
||||
2,
|
||||
"IMAGE"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 1.1,
|
||||
"offset": [
|
||||
-2768.063780855268,
|
||||
-838.8118837029729
|
||||
]
|
||||
},
|
||||
"workflowRendererVersion": "LG",
|
||||
"frontendVersion": "1.32.9",
|
||||
"VHS_latentpreview": false,
|
||||
"VHS_latentpreviewrate": 0,
|
||||
"VHS_MetadataImage": true,
|
||||
"VHS_KeepIntermediate": true
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
{
|
||||
"last_node_id": 12,
|
||||
"last_link_id": 13,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "UNETLoader",
|
||||
"pos": [-380, 260],
|
||||
"size": [340, 82],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"outputs": [{"name": "MODEL", "type": "MODEL", "links": [1]}],
|
||||
"properties": {"Node name for S&R": "UNETLoader"},
|
||||
"widgets_values": ["qwen_image_edit_2511_fp8mixed.safetensors", "fp8_e4m3fn"]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "CLIPLoader",
|
||||
"pos": [-380, 400],
|
||||
"size": [340, 82],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"outputs": [{"name": "CLIP", "type": "CLIP", "links": [3]}],
|
||||
"properties": {"Node name for S&R": "CLIPLoader"},
|
||||
"widgets_values": ["qwen_2.5_vl_7b_fp8_scaled.safetensors", "qwen_image"]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "VAELoader",
|
||||
"pos": [-380, 540],
|
||||
"size": [340, 58],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"outputs": [{"name": "VAE", "type": "VAE", "links": [4, 11]}],
|
||||
"properties": {"Node name for S&R": "VAELoader"},
|
||||
"widgets_values": ["qwen_image_vae.safetensors"]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "LoadImage",
|
||||
"pos": [-380, 660],
|
||||
"size": [340, 314],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{"name": "IMAGE", "type": "IMAGE", "links": [5]},
|
||||
{"name": "MASK", "type": "MASK", "links": null}
|
||||
],
|
||||
"properties": {"Node name for S&R": "LoadImage"},
|
||||
"widgets_values": ["example.png", "image"]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "TextEncodeQwenImageEditPlus",
|
||||
"pos": [40, 620],
|
||||
"size": [400, 260],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"name": "clip", "type": "CLIP", "link": 3},
|
||||
{"name": "vae", "type": "VAE", "link": 4},
|
||||
{"name": "image1", "type": "IMAGE", "link": 5},
|
||||
{"name": "image2", "type": "IMAGE", "link": null},
|
||||
{"name": "image3", "type": "IMAGE", "link": null}
|
||||
],
|
||||
"outputs": [{"name": "CONDITIONING", "type": "CONDITIONING", "links": [6, 7]}],
|
||||
"properties": {"Node name for S&R": "TextEncodeQwenImageEditPlus"},
|
||||
"widgets_values": ["Change the color of the rose from red to blue"]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "ConditioningZeroOut",
|
||||
"pos": [40, 940],
|
||||
"size": [300, 46],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [{"name": "conditioning", "type": "CONDITIONING", "link": 6}],
|
||||
"outputs": [{"name": "CONDITIONING", "type": "CONDITIONING", "links": [8]}],
|
||||
"properties": {"Node name for S&R": "ConditioningZeroOut"}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "GRAGPatch",
|
||||
"pos": [40, 260],
|
||||
"size": [380, 262],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [{"name": "model", "type": "MODEL", "link": 1}],
|
||||
"outputs": [{"name": "model", "type": "MODEL", "links": [2, 13]}],
|
||||
"properties": {"Node name for S&R": "GRAGPatch"},
|
||||
"widgets_values": [1.1, 1.0, 4096, 1.0, 1.0, 0.0, 1.0, false]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "GRAGInspect",
|
||||
"pos": [40, 60],
|
||||
"size": [380, 130],
|
||||
"flags": {},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [{"name": "model", "type": "MODEL", "link": 13}],
|
||||
"outputs": [{"name": "report", "type": "STRING", "links": null}],
|
||||
"properties": {"Node name for S&R": "GRAGInspect"},
|
||||
"widgets_values": [1024, 1024]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "EmptySD3LatentImage",
|
||||
"pos": [40, 1030],
|
||||
"size": [300, 106],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"outputs": [{"name": "LATENT", "type": "LATENT", "links": [9]}],
|
||||
"properties": {"Node name for S&R": "EmptySD3LatentImage"},
|
||||
"widgets_values": [1024, 1024, 1]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "KSampler",
|
||||
"pos": [520, 340],
|
||||
"size": [320, 262],
|
||||
"flags": {},
|
||||
"order": 9,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"name": "model", "type": "MODEL", "link": 2},
|
||||
{"name": "positive", "type": "CONDITIONING", "link": 7},
|
||||
{"name": "negative", "type": "CONDITIONING", "link": 8},
|
||||
{"name": "latent_image", "type": "LATENT", "link": 9}
|
||||
],
|
||||
"outputs": [{"name": "LATENT", "type": "LATENT", "links": [10]}],
|
||||
"properties": {"Node name for S&R": "KSampler"},
|
||||
"widgets_values": [42, "randomize", 24, 4.0, "euler", "simple", 1.0]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "VAEDecode",
|
||||
"pos": [900, 340],
|
||||
"size": [210, 46],
|
||||
"flags": {},
|
||||
"order": 10,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"name": "samples", "type": "LATENT", "link": 10},
|
||||
{"name": "vae", "type": "VAE", "link": 11}
|
||||
],
|
||||
"outputs": [{"name": "IMAGE", "type": "IMAGE", "links": [12]}],
|
||||
"properties": {"Node name for S&R": "VAEDecode"}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "SaveImage",
|
||||
"pos": [900, 440],
|
||||
"size": [380, 400],
|
||||
"flags": {},
|
||||
"order": 11,
|
||||
"mode": 0,
|
||||
"inputs": [{"name": "images", "type": "IMAGE", "link": 12}],
|
||||
"properties": {"Node name for S&R": "SaveImage"},
|
||||
"widgets_values": ["GRAG_2511"]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[1, 1, 0, 7, 0, "MODEL"],
|
||||
[2, 7, 0, 9, 0, "MODEL"],
|
||||
[3, 2, 0, 5, 0, "CLIP"],
|
||||
[4, 3, 0, 5, 1, "VAE"],
|
||||
[5, 4, 0, 5, 2, "IMAGE"],
|
||||
[6, 5, 0, 6, 0, "CONDITIONING"],
|
||||
[7, 5, 0, 9, 1, "CONDITIONING"],
|
||||
[8, 6, 0, 9, 2, "CONDITIONING"],
|
||||
[9, 8, 0, 9, 3, "LATENT"],
|
||||
[10, 9, 0, 10, 0, "LATENT"],
|
||||
[11, 3, 0, 10, 1, "VAE"],
|
||||
[12, 10, 0, 11, 0, "IMAGE"],
|
||||
[13, 7, 0, 12, 0, "MODEL"]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"note": "GRAG on Qwen-Image-Edit-2511, official params (24 steps, cfg 4.0, cond_delta 1.10). 2511 REQUIRES TextEncodeQwenImageEditPlus + ConditioningZeroOut negative. For the 4-step Lightning lora, insert LoraLoaderModelOnly between UNETLoader and GRAGPatch and use cfg 1.0 / steps 4. GRAGPatch: cond_delta < 1 = stronger edit, > 1 = lock onto the reference; 1.0/1.0 = exact no-op."
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
+279
@@ -0,0 +1,279 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
"""GRAG (Group-Relative Attention Guidance) — native comfy DiT core implementation.
|
||||
|
||||
Reference implementation: https://github.com/little-misfit/GRAG-Image-Editing
|
||||
Paper: arXiv 2510.24657 "Group Relative Attention Guidance for Image Editing"
|
||||
|
||||
------------------------------------------------------------------------------
|
||||
1. What upstream actually does
|
||||
------------------------------------------------------------------------------
|
||||
``hacked_models/models.py::QwenDoubleStreamAttnProcessor2_0.__call__``::
|
||||
|
||||
img_query/key/value = to_q/k/v(hidden_states) # image stream
|
||||
txt_query/key/value = add_q/k/v(encoder_hidden) # text stream
|
||||
... norm_q / norm_k / norm_added_q / norm_added_k ...
|
||||
img_q,k = apply_rotary_emb_qwen(..., img_freqs) # <-- RoPE here
|
||||
txt_q,k = apply_rotary_emb_qwen(..., txt_freqs)
|
||||
|
||||
if grag_scale != (0, 0):
|
||||
txt_len, txt_bias, txt_delta = txt_srag_scale # ((512, 1.0, 1.0))
|
||||
img_len, img_bias, img_delta = cond_srag_scale # ((4096, cond_b, cond_delta))
|
||||
txt_key_mean = txt_key[:, :txt_len, :, :].mean(dim=1) # [B, H, D]
|
||||
cond_key_mean = img_key[:, -img_len:, :, :].mean(dim=1)
|
||||
txt_key[:, :txt_len, :, :] = txt_bias * txt_key_mean + (txt_key - txt_key_mean) * txt_delta
|
||||
img_key[:, -img_len:, :, :] = img_bias * cond_key_mean + (img_key - cond_key_mean) * img_delta
|
||||
|
||||
joint_q = cat([txt_q, img_q], dim=1) # text first, image after
|
||||
...
|
||||
|
||||
i.e. GRAG rewrites the **post-RoPE** joint key as
|
||||
|
||||
k' = bias * mean(k_group) + (k - mean(k_group)) * delta
|
||||
|
||||
The paper tunes the *image* group (``cond_b`` / ``cond_delta``); the text group is
|
||||
left at ``(1.0, 1.0)`` which is an exact identity.
|
||||
|
||||
------------------------------------------------------------------------------
|
||||
2. Where it hooks into comfy without touching a single core file
|
||||
------------------------------------------------------------------------------
|
||||
``comfy/ldm/modules/attention.py::wrap_attn`` calls
|
||||
``transformer_options["optimized_attention_override"](func, q, k, v, heads, **kw)``
|
||||
before dispatch (the same route ``comfy_extras/nodes_sparse_attention.py`` uses).
|
||||
At that point q/k/v are already QK-normed **and RoPE'd**, in ``[B, H, S, D]`` with
|
||||
``skip_reshape=True`` — which is exactly the tensor upstream reweights, so the port
|
||||
is bit-exact with no RoPE replication needed.
|
||||
|
||||
Measured on this machine (``_grag_override_probe5.py``, random weights so the
|
||||
rotation is observable)::
|
||||
|
||||
dmax(captured_k , pre-rope k) = 8.46875
|
||||
dmax(captured_k , post-rope k) = 0.0
|
||||
=> override sees POST-rope keys
|
||||
|
||||
The text/image split is not in ``transformer_options`` at that point (comfy writes
|
||||
``img_slice`` into a *local* copy for ``attn1_patch`` only), so a tiny
|
||||
``attn1_patch`` recorder caches ``seq_txt`` from the immediately preceding
|
||||
``attn1_patch`` call of the same attention block. Recorder and override are strictly
|
||||
sequential inside ``Attention.forward``, and a tick/consume pair makes a stale value
|
||||
impossible: if the override runs without a fresh tick it degrades to a no-op.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# core math
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
def grag_reweight(key, txt_len, img_len, lambda_val, delta_val,
|
||||
txt_lambda=1.0, txt_delta=1.0):
|
||||
"""Group-relative reweighting of a joint attention key ``[B, H, S, D]``.
|
||||
|
||||
``txt_len`` : number of leading text tokens (sequence is ``[text | image]``).
|
||||
``img_len`` : how many *trailing* image tokens to reweight, 0 = the whole group.
|
||||
Upstream hardcodes 4096; at 1024² a qwen edit grid is exactly 4096
|
||||
tokens so the default covers everything, and the clamp below keeps
|
||||
larger grids from silently reaching into the text part.
|
||||
``lambda_val`` / ``delta_val`` : image-group bias / deviation scale (cond_b, cond_delta).
|
||||
``txt_lambda`` / ``txt_delta`` : text-group pair, upstream leaves it at (1, 1).
|
||||
|
||||
All of it is pure elementwise arithmetic on already-materialised q/k/v, so it
|
||||
costs two means and two FMA passes per block per step.
|
||||
"""
|
||||
txt = key[:, :, :txt_len, :]
|
||||
img = key[:, :, txt_len:, :]
|
||||
|
||||
if img_len and img_len > 0 and img_len < img.shape[2]:
|
||||
img_head = img[:, :, :img.shape[2] - img_len, :]
|
||||
img_tail = img[:, :, img.shape[2] - img_len:, :]
|
||||
else:
|
||||
img_head = None
|
||||
img_tail = img
|
||||
|
||||
if lambda_val != 1.0 or delta_val != 1.0:
|
||||
img_mean = img_tail.mean(dim=2, keepdim=True)
|
||||
img_tail = lambda_val * img_mean + (img_tail - img_mean) * delta_val
|
||||
|
||||
if txt_lambda != 1.0 or txt_delta != 1.0:
|
||||
txt_mean = txt.mean(dim=2, keepdim=True)
|
||||
txt = txt_lambda * txt_mean + (txt - txt_mean) * txt_delta
|
||||
|
||||
if img_head is None:
|
||||
return torch.cat([txt, img_tail], dim=2)
|
||||
return torch.cat([txt, img_head, img_tail], dim=2)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# shared state
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
class GRAGState:
|
||||
"""One instance per patched MODEL; mutated per step, read by the hooks."""
|
||||
|
||||
def __init__(self, lambda_val=1.0, delta_val=1.10, img_len=4096,
|
||||
txt_lambda=1.0, txt_delta=1.0, window=(0.0, 1.0)):
|
||||
self.lambda_val = float(lambda_val)
|
||||
self.delta_val = float(delta_val)
|
||||
self.img_len = int(img_len)
|
||||
self.txt_lambda = float(txt_lambda)
|
||||
self.txt_delta = float(txt_delta)
|
||||
# normalised progress window (0 = pure noise, 1 = clean)
|
||||
self.window = (min(window), max(window))
|
||||
|
||||
self.enabled = True # toggled by the step wrapper (sigma window)
|
||||
self.step = 0
|
||||
self.sigma = None
|
||||
self.txt_len = None # written by the attn1 recorder
|
||||
self._tick = 0 # recorder increments
|
||||
self._consumed = 0 # override reads + consumes
|
||||
|
||||
self.blocks_fired = 0
|
||||
|
||||
# -- recorder side ---------------------------------------------------------
|
||||
def note_txt_len(self, txt_len):
|
||||
self.txt_len = int(txt_len)
|
||||
self._tick += 1
|
||||
|
||||
def fresh_txt_len(self):
|
||||
"""Return a txt_len that arrived after the last override call, else None."""
|
||||
if self._tick == self._consumed or self.txt_len is None:
|
||||
return None
|
||||
self._consumed = self._tick
|
||||
return self.txt_len
|
||||
|
||||
# -- gating ----------------------------------------------------------------
|
||||
def math_active(self):
|
||||
"""True when the user asked for a non-identity transform."""
|
||||
return (self.lambda_val != 1.0 or self.delta_val != 1.0
|
||||
or self.txt_lambda != 1.0 or self.txt_delta != 1.0)
|
||||
|
||||
def active(self):
|
||||
return self.enabled and self.math_active()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# hooks
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
def make_attn1_recorder(state):
|
||||
"""``attn1_patch`` that only caches ``seq_txt`` (does not touch q/k/v).
|
||||
|
||||
comfy calls it as ``patch(q, k, v, pe=..., attn_mask=..., extra_options=...)``
|
||||
with ``extra_options["img_slice"] = [seq_txt, total]``, immediately before the
|
||||
RoPE + ``optimized_attention`` of the very same block.
|
||||
"""
|
||||
|
||||
def recorder(q, k, v, pe=None, attn_mask=None, extra_options=None):
|
||||
eo = extra_options or {}
|
||||
img_slice = eo.get("img_slice")
|
||||
if img_slice is not None:
|
||||
state.note_txt_len(int(img_slice[0]))
|
||||
return {}
|
||||
|
||||
recorder.is_grag_recorder = True
|
||||
return recorder
|
||||
|
||||
|
||||
def make_attention_override(state, previous=None):
|
||||
"""``optimized_attention_override`` that applies GRAG then delegates to comfy.
|
||||
|
||||
``func`` is the selected attention kernel's inner function, so plain delegation
|
||||
keeps whatever backend the launch flags picked. An override installed earlier in
|
||||
the chain is chained through, so other MODEL patches keep working.
|
||||
"""
|
||||
|
||||
def override(func, *args, **kwargs):
|
||||
passthrough = (previous, func, args, kwargs) if previous is not None else (None, func, args, kwargs)
|
||||
|
||||
if not state.active() or len(args) < 4:
|
||||
return _delegate(passthrough)
|
||||
|
||||
q, k, v = args[0], args[1], args[2]
|
||||
if not (torch.is_tensor(q) and torch.is_tensor(k) and torch.is_tensor(v)):
|
||||
return _delegate(passthrough)
|
||||
|
||||
txt_len = state.fresh_txt_len()
|
||||
if txt_len is None or txt_len <= 0 or txt_len >= k.shape[2]:
|
||||
return _delegate(passthrough)
|
||||
|
||||
new_k = grag_reweight(
|
||||
k, txt_len, state.img_len,
|
||||
state.lambda_val, state.delta_val,
|
||||
state.txt_lambda, state.txt_delta,
|
||||
)
|
||||
state.blocks_fired += 1
|
||||
return _delegate((previous, func, (q, new_k, v) + tuple(args[3:]), kwargs))
|
||||
|
||||
override.is_grag = True
|
||||
return override
|
||||
|
||||
|
||||
def _delegate(payload):
|
||||
previous, func, args, kwargs = payload
|
||||
if previous is not None:
|
||||
return previous(func, *args, **kwargs)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
|
||||
def install_hooks(patcher, state):
|
||||
"""Attach recorder + override to a cloned ModelPatcher (idempotent)."""
|
||||
to = patcher.model_options.setdefault("transformer_options", {})
|
||||
|
||||
patches = to.setdefault("patches", {})
|
||||
existing = patches.get("attn1_patch") or []
|
||||
if not any(getattr(p, "is_grag_recorder", False) for p in existing):
|
||||
patches["attn1_patch"] = list(existing) + [make_attn1_recorder(state)]
|
||||
|
||||
if state.math_active():
|
||||
current = to.get("optimized_attention_override")
|
||||
if not getattr(current, "is_grag", False):
|
||||
to["optimized_attention_override"] = make_attention_override(state, previous=current)
|
||||
|
||||
return patcher
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# per-step gate
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
def make_step_wrapper(state, total_steps=None, start_percent=0.0, end_percent=1.0, previous=None):
|
||||
"""Model-function wrapper that reports progress and gates GRAG by sigma.
|
||||
|
||||
``params`` is the dict comfy builds in ``samplers.py``:
|
||||
``{"input", "timestep", "c", "cond_or_uncond"}``; ``timestep`` is the current
|
||||
sigma (flow matching, ``ModelSamplingFlux.timestep(σ) == σ``).
|
||||
|
||||
The gate exists because a 4-step distilled LoRA spends most of its budget below
|
||||
σ≈0.3, where the edit is already committed and reweighting only adds noise. The
|
||||
default window is the full run, which reproduces upstream exactly.
|
||||
"""
|
||||
lo, hi = state.window
|
||||
|
||||
def wrapper(model_function, params):
|
||||
sigma = params.get("timestep")
|
||||
if torch.is_tensor(sigma) and sigma.numel() == 1:
|
||||
s = float(sigma.reshape(-1)[0].item())
|
||||
state.sigma = s
|
||||
if total_steps and total_steps > 1:
|
||||
state.step = int(round((1.0 - s) * (total_steps - 1)))
|
||||
progress = 1.0 - s
|
||||
state.enabled = (lo <= progress <= hi) if (lo > 0.0 or hi < 1.0) else True
|
||||
if previous is not None:
|
||||
return previous(model_function, params)
|
||||
return model_function(params["input"], params["timestep"], **params["c"])
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def install_step_wrapper(patcher, state, total_steps=None, window=(0.0, 1.0)):
|
||||
state.window = (min(window), max(window))
|
||||
if state.window == (0.0, 1.0):
|
||||
# nothing to gate: leave any pre-existing wrapper untouched
|
||||
return
|
||||
previous = patcher.model_options.get("model_function_wrapper")
|
||||
patcher.set_model_unet_function_wrapper(
|
||||
make_step_wrapper(state, total_steps=total_steps,
|
||||
start_percent=state.window[0], end_percent=state.window[1],
|
||||
previous=previous)
|
||||
)
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
"""GRAG nodes on the native comfy DiT stack.
|
||||
"""
|
||||
|
||||
from comfy_api.latest import io
|
||||
from . import grag_core
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# nodes
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
class GRAGPatch(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="GRAGPatch",
|
||||
display_name="GRAG Patch (Group-Relative Attention Guidance)",
|
||||
category="GRAG",
|
||||
description=(
|
||||
"Group-Relative Attention Guidance (arXiv 2510.24657) on a native comfy qwen-image-edit "
|
||||
"model. No weights, VAE, CLIP or sampler are replaced: the node only rewrites the joint "
|
||||
"attention **keys** after RoPE.\n"
|
||||
"\n"
|
||||
"k' = cond_b * mean(k_ref) + (k_ref - mean(k_ref)) * cond_delta\n"
|
||||
"\n"
|
||||
"The paper splits each key into a shared, layer-only bias (the model's intrinsic editing "
|
||||
"behaviour) plus a per-token delta (the content-specific edit signal). Re-weighting the "
|
||||
"delta modulates how strongly the model follows the edit versus how strongly it holds on "
|
||||
"to the reference image.\n"
|
||||
"\n"
|
||||
"cond_delta < 1 : the per-token deviation shrinks, the reference group's attention "
|
||||
"signal flattens and the model is freed from the reference -> stronger edit.\n"
|
||||
"cond_delta > 1 : the deviation is amplified -> the output locks onto the reference.\n"
|
||||
"Verified on Qwen-Image-Edit-2511 (cfg 4.0, 24 steps): delta=1.70 -> corr 0.995 with the "
|
||||
"input (near copy), delta=1.10 -> corr 0.997 (mild), delta=0.90 -> corr 0.928, "
|
||||
"delta=0.80 -> corr 0.711 (image drifts far from the reference).\n"
|
||||
"Recommended sweep (upstream): 0.8 ~ 1.7 in 0.01 steps, default 1.10."
|
||||
"cond_b (upstream 'bias scale') is kept at 1.0 in every released example - leave it alone "
|
||||
"unless you are deliberately deviating from the paper.\n"
|
||||
"\n"
|
||||
"cond_b == 1.0 and cond_delta == 1.0 is a bit-exact no-op, so the node is safe to leave "
|
||||
"wired in."
|
||||
),
|
||||
inputs=[
|
||||
io.Model.Input("model"),
|
||||
io.Float.Input("cond_delta", default=1.10, min=0.0, max=4.0, step=0.01,
|
||||
tooltip="Deviation scale of the reference-image key group. <1 flattens the "
|
||||
"group signal -> the edit gets stronger (reference holds less); "
|
||||
">1 amplifies it -> the output locks onto the reference image. "
|
||||
"1.0 = disabled. Upstream default 1.10."),
|
||||
io.Float.Input("cond_b", default=1.0, min=0.0, max=4.0, step=0.01,
|
||||
tooltip="Bias scale (lambda) of the reference-image group. Upstream keeps this "
|
||||
"at 1.0 everywhere; changing it rescales the shared edit-behaviour "
|
||||
"vector instead of the per-token signal."),
|
||||
io.Int.Input("img_len", default=4096, min=0, max=65536, step=16,
|
||||
tooltip="How many trailing image tokens form the 'reference group'. Upstream "
|
||||
"hardcodes 4096 (a 1024x1024 grid). 0 = the entire image block, which is "
|
||||
"what you want above 1024x1024."),
|
||||
io.Float.Input("txt_delta", default=1.0, min=0.0, max=4.0, step=0.01,
|
||||
tooltip="Deviation scale of the text group. Upstream sets it to 1.0 (no-op); "
|
||||
"<1 dampens the instruction stream, >1 amplifies it."),
|
||||
io.Float.Input("txt_b", default=1.0, min=0.0, max=4.0, step=0.01,
|
||||
tooltip="Bias scale of the text group (upstream: 1.0)."),
|
||||
io.Float.Input("apply_from", default=0.0, min=0.0, max=1.0, step=0.01,
|
||||
tooltip="Start of the sigma window, as denoising progress (0 = pure noise). "
|
||||
"0.0 = from the first step, matching upstream."),
|
||||
io.Float.Input("apply_to", default=1.0, min=0.0, max=1.0, step=0.01,
|
||||
tooltip="End of the sigma window. A 4-step distilled LoRA often looks better "
|
||||
"with something like 0.0 -> 0.6 because late steps are already "
|
||||
"committed. 1.0 = to the last step, matching upstream."),
|
||||
io.Boolean.Input("verbose", default=False, optional=True),
|
||||
],
|
||||
outputs=[io.Model.Output(display_name="model")],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, model, cond_delta, cond_b, img_len, txt_delta, txt_b,
|
||||
apply_from, apply_to, verbose=False) -> io.NodeOutput:
|
||||
state = grag_core.GRAGState(
|
||||
lambda_val=cond_b,
|
||||
delta_val=cond_delta,
|
||||
img_len=img_len,
|
||||
txt_lambda=txt_b,
|
||||
txt_delta=txt_delta,
|
||||
window=(apply_from, apply_to),
|
||||
)
|
||||
|
||||
if not state.math_active():
|
||||
# Exact identity: hand the MODEL straight back, no clone and no hooks.
|
||||
# Cloning a ModelPatcher is not free of side effects on comfy's model
|
||||
# management (a clone is tracked separately and can be loaded/streamed
|
||||
# differently), and an attn1_patch entry also changes what
|
||||
# merge_nested_dicts/cast_transformer_options walk - both showed up as a
|
||||
# visible dmax in the A/B even though GRAG itself is a no-op here.
|
||||
if verbose:
|
||||
print("[GRAG] cond_b=1.0 and cond_delta=1.0 -> pass-through, nothing installed")
|
||||
return io.NodeOutput(model)
|
||||
|
||||
m = model.clone()
|
||||
grag_core.install_hooks(m, state)
|
||||
grag_core.install_step_wrapper(m, state, window=(apply_from, apply_to))
|
||||
|
||||
# stash so GRAGInspect can read the effective config back
|
||||
m.model_options.setdefault("grag_state", state)
|
||||
|
||||
if verbose:
|
||||
print(f"[GRAG] patched: cond_b={cond_b:.3f} cond_delta={cond_delta:.3f} "
|
||||
f"img_len={img_len} txt=({txt_b:.3f},{txt_delta:.3f}) "
|
||||
f"window={apply_from:.2f}..{apply_to:.2f}")
|
||||
return io.NodeOutput(m)
|
||||
|
||||
@@ -1,422 +0,0 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import os
|
||||
import torch
|
||||
import gc
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
import math
|
||||
import comfy.utils
|
||||
import cv2
|
||||
import folder_paths
|
||||
from comfy.utils import common_upscale,ProgressBar
|
||||
from safetensors.torch import load_file
|
||||
cur_path = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
|
||||
def get_emb_data(clip,vae,prompt,image_list,ng=False,img=None,plus=False) : #image_list[ tensor,]
|
||||
ref_latents = None
|
||||
if image_list is None :
|
||||
images = []
|
||||
elif image_list is not None and not ng:
|
||||
images,ref_latents=get_image(vae,image_list,plus)
|
||||
else:
|
||||
samples = img.movedim(-1, 1)
|
||||
total = int(1024 * 1024)
|
||||
|
||||
scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
|
||||
width = round(samples.shape[3] * scale_by)
|
||||
height = round(samples.shape[2] * scale_by)
|
||||
|
||||
s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
|
||||
image = s.movedim(1, -1)
|
||||
images = [image[:, :, :, :3]]
|
||||
tokens = clip.tokenize(prompt,images=images)
|
||||
conditioning = clip.encode_from_tokens_scheduled(tokens,)
|
||||
|
||||
return conditioning,ref_latents
|
||||
|
||||
def get_image(vae,imgs,plus=False):
|
||||
ref_latents = None
|
||||
|
||||
if plus:
|
||||
images = []
|
||||
ref_latents = []
|
||||
for img in imgs:
|
||||
|
||||
samples = img.movedim(-1, 1)
|
||||
total = int(1024 * 1024)
|
||||
|
||||
scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
|
||||
width = round(samples.shape[3] * scale_by)
|
||||
height = round(samples.shape[2] * scale_by)
|
||||
|
||||
s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
|
||||
image = s.movedim(1, -1)
|
||||
images.append(image[:, :, :, :3])
|
||||
#images = [image[:, :, :, :3]]
|
||||
if vae is not None:
|
||||
ref_lat = vae.encode(image[:, :, :, :3])
|
||||
ref_latents.append(ref_lat)
|
||||
else:
|
||||
image=imgs[0]
|
||||
samples = image.movedim(-1, 1)
|
||||
total = int(1024 * 1024)
|
||||
|
||||
scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
|
||||
width = round(samples.shape[3] * scale_by)
|
||||
height = round(samples.shape[2] * scale_by)
|
||||
|
||||
s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
|
||||
image = s.movedim(1, -1)
|
||||
images = [image[:, :, :, :3]]
|
||||
if vae is not None:
|
||||
ref_latents = vae.encode(image[:, :, :, :3])
|
||||
|
||||
return images,ref_latents
|
||||
|
||||
|
||||
|
||||
def encode_image( image, vae):
|
||||
if image is None:
|
||||
return None
|
||||
ref_latents=None
|
||||
samples = image.movedim(-1, 1)
|
||||
total = int(1024 * 1024)
|
||||
scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
|
||||
width = round(samples.shape[3] * scale_by)
|
||||
height = round(samples.shape[2] * scale_by)
|
||||
|
||||
s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
|
||||
image = s.movedim(1, -1)
|
||||
if vae is not None:
|
||||
ref_latents = vae.encode(image[:, :, :, :3])
|
||||
return ref_latents
|
||||
|
||||
def add_mean(latents):
|
||||
vae_config={"latents_mean": [
|
||||
-0.7571,
|
||||
-0.7089,
|
||||
-0.9113,
|
||||
0.1075,
|
||||
-0.1745,
|
||||
0.9653,
|
||||
-0.1517,
|
||||
1.5508,
|
||||
0.4134,
|
||||
-0.0715,
|
||||
0.5517,
|
||||
-0.3632,
|
||||
-0.1922,
|
||||
-0.9497,
|
||||
0.2503,
|
||||
-0.2921
|
||||
],
|
||||
"latents_std": [
|
||||
2.8184,
|
||||
1.4541,
|
||||
2.3275,
|
||||
2.6558,
|
||||
1.2196,
|
||||
1.7708,
|
||||
2.6052,
|
||||
2.0743,
|
||||
3.2687,
|
||||
2.1526,
|
||||
2.8652,
|
||||
1.5579,
|
||||
1.6382,
|
||||
1.1253,
|
||||
2.8251,
|
||||
1.916
|
||||
],}
|
||||
latents_mean = (torch.tensor(vae_config["latents_mean"]).view(1, 16, 1, 1, 1).to(latents.device, latents.dtype))
|
||||
latents_std = 1.0 / torch.tensor(vae_config["latents_std"]).view(1, 16, 1, 1, 1).to(latents.device, latents.dtype)
|
||||
latents = latents / latents_std + latents_mean
|
||||
image_latent_height, image_latent_width = latents.shape[3:]
|
||||
image_latents = pack_latents_(
|
||||
latents, 1, 16, image_latent_height, image_latent_width)
|
||||
return image_latents
|
||||
|
||||
def pack_latents_(latents, batch_size, num_channels_latents, height, width):
|
||||
latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
|
||||
latents = latents.permute(0, 2, 4, 1, 3, 5)
|
||||
latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
|
||||
return latents
|
||||
|
||||
|
||||
def load_lora(model, lora_1, lora_2, lora_scale1, lora_scale2):
|
||||
lora_path_1=folder_paths.get_full_path("loras", lora_1) if lora_1 != "none" else None
|
||||
lora_path_2=folder_paths.get_full_path("loras", lora_2) if lora_2 != "none" else None
|
||||
# lora_list=[i for i in [lora_path_1,lora_path_2] if i is not None]
|
||||
# lora_scales=[lora_scale1,lora_scale2]
|
||||
all_adapters = model.get_list_adapters()
|
||||
dit_list=[]
|
||||
if all_adapters:
|
||||
dit_list= all_adapters.get('transformer',[])+all_adapters.get('transformer_2',[])
|
||||
if lora_path_1 is not None:
|
||||
adapter_name=os.path.splitext(os.path.basename(lora_path_1))[0].replace(".", "_")
|
||||
dit_list2=all_adapters.get('transformer_2',[])
|
||||
if dit_list2:
|
||||
if adapter_name in dit_list: #dit_list
|
||||
pass
|
||||
else:
|
||||
for i in dit_list2:
|
||||
model.delete_adapters(i)
|
||||
print(f"去除dit中未加载的lora: {i}")
|
||||
try:
|
||||
model.load_lora_weights(lora_path_1, adapter_name=adapter_name,**{"load_into_transformer_2": True})
|
||||
model.set_adapters([adapter_name], adapter_weights=lora_scale1)
|
||||
except KeyError as e:
|
||||
try:
|
||||
print(f"检测到特殊的 LoRA 格式，尝试手动处理: {lora_path_1}")
|
||||
state_dict = torch.load(lora_path_1, map_location="cpu",weights_only=False) if not lora_path_1.endswith(".safetensors") else load_file(lora_path_1,)
|
||||
processed_state_dict = preprocess_lora_state_dict(state_dict)
|
||||
model.load_lora_weights(processed_state_dict, adapter_name=adapter_name,**{"load_into_transformer_2": True})
|
||||
model.set_adapters([adapter_name], adapter_weights=lora_scale1)
|
||||
except:
|
||||
print(f"加载LoRA权重失败: {e}")
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
model.load_lora_weights(lora_path_1, adapter_name=adapter_name,**{"load_into_transformer_2": True})
|
||||
model.set_adapters([adapter_name], adapter_weights=lora_scale1)
|
||||
except KeyError as e:
|
||||
try:
|
||||
print(f"检测到特殊的 LoRA 格式，尝试手动处理: {lora_path_1}")
|
||||
state_dict = torch.load(lora_path_1, map_location="cpu",weights_only=False) if not lora_path_1.endswith(".safetensors") else load_file(lora_path_1,)
|
||||
processed_state_dict = preprocess_lora_state_dict(state_dict)
|
||||
model.load_lora_weights(processed_state_dict, adapter_name=adapter_name,**{"load_into_transformer_2": True})
|
||||
model.set_adapters([adapter_name], adapter_weights=lora_scale1)
|
||||
del processed_state_dict
|
||||
except:
|
||||
print(f"加载LoRA权重失败: {e}")
|
||||
pass
|
||||
if lora_path_2 is not None:
|
||||
adapter_name=os.path.splitext(os.path.basename(lora_path_2))[0].replace(".", "_")
|
||||
dit_list=all_adapters.get('transformer',[])
|
||||
if dit_list:
|
||||
if adapter_name in dit_list: #dit_list
|
||||
pass
|
||||
else:
|
||||
for i in dit_list:
|
||||
model.delete_adapters(i)
|
||||
print(f"去除dit中未加载的lora: {i}")
|
||||
try:
|
||||
model.load_lora_weights(lora_path_2, adapter_name=adapter_name,**{"load_into_transformer_2": False})
|
||||
model.set_adapters([adapter_name], adapter_weights=lora_scale2)
|
||||
except KeyError as e:
|
||||
try:
|
||||
print(f"检测到特殊的 LoRA 格式，尝试手动处理: {lora_path_2}")
|
||||
state_dict = torch.load(lora_path_2, map_location="cpu",weights_only=False) if not lora_path_2.endswith(".safetensors") else load_file(lora_path_2,)
|
||||
processed_state_dict = preprocess_lora_state_dict(state_dict)
|
||||
model.load_lora_weights(processed_state_dict, adapter_name=adapter_name,**{"load_into_transformer_2": False})
|
||||
model.set_adapters([adapter_name], adapter_weights=lora_scale2)
|
||||
del processed_state_dict
|
||||
except:
|
||||
print(f"加载LoRA权重失败: {e}")
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
model.load_lora_weights(lora_path_2, adapter_name=adapter_name,**{"load_into_transformer_2": False})
|
||||
model.set_adapters([adapter_name], adapter_weights=lora_scale2)
|
||||
except KeyError as e:
|
||||
try:
|
||||
print(f"检测到特殊的 LoRA 格式，尝试手动处理: {lora_path_2}")
|
||||
state_dict = torch.load(lora_path_2, map_location="cpu",weights_only=False) if not lora_path_2.endswith(".safetensors") else load_file(lora_path_2,)
|
||||
processed_state_dict = preprocess_lora_state_dict(state_dict)
|
||||
model.load_lora_weights(processed_state_dict, adapter_name=adapter_name,**{"load_into_transformer_2": False})
|
||||
model.set_adapters([adapter_name], adapter_weights=lora_scale2)
|
||||
del processed_state_dict
|
||||
except:
|
||||
print(f"加载LoRA权重失败: {e}")
|
||||
pass
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def preprocess_lora_state_dict(state_dict):
|
||||
|
||||
processed_dict = state_dict.copy()
|
||||
keys_to_remove = [
|
||||
'head.head.diff_b',
|
||||
'head.head.diff_m',
|
||||
'head.head.diff',
|
||||
'patch_embedding.diff',
|
||||
'patch_embedding.diff_b',
|
||||
'blocks.*.diff_m', # 匹配所有blocks的diff_m
|
||||
'head.head.lora_down'
|
||||
'diffusion_model.head.head.diff'
|
||||
'diffusion_model.head.head.diff_b'
|
||||
'diffusion_model.head.lora_down'
|
||||
|
||||
]
|
||||
keys_to_delete = []
|
||||
for key in processed_dict.keys():
|
||||
if key.endswith('.diff_m'):
|
||||
keys_to_delete.append(key)
|
||||
for key in keys_to_delete:
|
||||
processed_dict.pop(key, None)
|
||||
print(f"移除键: {key}")
|
||||
for key in keys_to_remove:
|
||||
if key in processed_dict:
|
||||
processed_dict.pop(key, None)
|
||||
print(f"移除键: {key}")
|
||||
return processed_dict
|
||||
|
||||
def gc_cleanup():
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
def tensor2cv(tensor_image):
|
||||
if len(tensor_image.shape)==4:# b hwc to hwc
|
||||
tensor_image=tensor_image.squeeze(0)
|
||||
if tensor_image.is_cuda:
|
||||
tensor_image = tensor_image.cpu()
|
||||
tensor_image=tensor_image.numpy()
|
||||
#反归一化
|
||||
maxValue=tensor_image.max()
|
||||
tensor_image=tensor_image*255/maxValue
|
||||
img_cv2=np.uint8(tensor_image)#32 to uint8
|
||||
img_cv2=cv2.cvtColor(img_cv2,cv2.COLOR_RGB2BGR)
|
||||
return img_cv2
|
||||
|
||||
def phi2narry(img):
|
||||
img = torch.from_numpy(np.array(img).astype(np.float32) / 255.0).unsqueeze(0)
|
||||
return img
|
||||
|
||||
def tensor2image(tensor):
|
||||
tensor = tensor.cpu()
|
||||
image_np = tensor.squeeze().mul(255).clamp(0, 255).byte().numpy()
|
||||
image = Image.fromarray(image_np, mode='RGB')
|
||||
return image
|
||||
|
||||
def tensor2pillist(tensor_in):
|
||||
d1, _, _, _ = tensor_in.size()
|
||||
if d1 == 1:
|
||||
img_list = [tensor2image(tensor_in)]
|
||||
else:
|
||||
tensor_list = torch.chunk(tensor_in, chunks=d1)
|
||||
img_list=[tensor2image(i) for i in tensor_list]
|
||||
return img_list
|
||||
|
||||
def tensor2pillist_upscale(tensor_in,width,height):
|
||||
d1, _, _, _ = tensor_in.size()
|
||||
if d1 == 1:
|
||||
img_list = [nomarl_upscale(tensor_in,width,height)]
|
||||
else:
|
||||
tensor_list = torch.chunk(tensor_in, chunks=d1)
|
||||
img_list=[nomarl_upscale(i,width,height) for i in tensor_list]
|
||||
return img_list
|
||||
|
||||
def tensor2list(tensor_in,width,height):
|
||||
if tensor_in is None:
|
||||
return None
|
||||
d1, _, _, _ = tensor_in.size()
|
||||
if d1 == 1:
|
||||
tensor_list = [tensor_upscale(tensor_in,width,height)]
|
||||
else:
|
||||
tensor_list_ = torch.chunk(tensor_in, chunks=d1)
|
||||
tensor_list=[tensor_upscale(i,width,height) for i in tensor_list_]
|
||||
return tensor_list
|
||||
|
||||
|
||||
def tensor_upscale(tensor, width, height):
|
||||
samples = tensor.movedim(-1, 1)
|
||||
samples = common_upscale(samples, width, height, "bilinear", "center")
|
||||
samples = samples.movedim(1, -1)
|
||||
return samples
|
||||
|
||||
def nomarl_upscale(img, width, height):
|
||||
samples = img.movedim(-1, 1)
|
||||
img = common_upscale(samples, width, height, "bilinear", "center")
|
||||
samples = img.movedim(1, -1)
|
||||
img = tensor2image(samples)
|
||||
return img
|
||||
|
||||
|
||||
|
||||
def cv2tensor(img,bgr2rgb=True):
|
||||
assert type(img) == np.ndarray, 'the img type is {}, but ndarry expected'.format(type(img))
|
||||
if bgr2rgb:
|
||||
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
||||
img = torch.from_numpy(img.transpose((2, 0, 1)))
|
||||
return img.float().div(255).permute(1, 2, 0).unsqueeze(0)
|
||||
|
||||
|
||||
|
||||
def images_generator(img_list: list, ):
|
||||
# get img size
|
||||
sizes = {}
|
||||
for image_ in img_list:
|
||||
if isinstance(image_, Image.Image):
|
||||
count = sizes.get(image_.size, 0)
|
||||
sizes[image_.size] = count + 1
|
||||
elif isinstance(image_, np.ndarray):
|
||||
count = sizes.get(image_.shape[:2][::-1], 0)
|
||||
sizes[image_.shape[:2][::-1]] = count + 1
|
||||
else:
|
||||
raise "unsupport image list,must be pil or cv2!!!"
|
||||
size = max(sizes.items(), key=lambda x: x[1])[0]
|
||||
yield size[0], size[1]
|
||||
|
||||
# any to tensor
|
||||
def load_image(img_in):
|
||||
if isinstance(img_in, Image.Image):
|
||||
img_in = img_in.convert("RGB")
|
||||
i = np.array(img_in, dtype=np.float32)
|
||||
i = torch.from_numpy(i).div_(255)
|
||||
if i.shape[0] != size[1] or i.shape[1] != size[0]:
|
||||
i = torch.from_numpy(i).movedim(-1, 0).unsqueeze(0)
|
||||
i = common_upscale(i, size[0], size[1], "lanczos", "center")
|
||||
i = i.squeeze(0).movedim(0, -1).numpy()
|
||||
return i
|
||||
elif isinstance(img_in, np.ndarray):
|
||||
i = cv2.cvtColor(img_in, cv2.COLOR_BGR2RGB).astype(np.float32)
|
||||
i = torch.from_numpy(i).div_(255)
|
||||
print(i.shape)
|
||||
return i
|
||||
else:
|
||||
raise "unsupport image list,must be pil,cv2 or tensor!!!"
|
||||
|
||||
total_images = len(img_list)
|
||||
processed_images = 0
|
||||
pbar = ProgressBar(total_images)
|
||||
images = map(load_image, img_list)
|
||||
try:
|
||||
prev_image = next(images)
|
||||
while True:
|
||||
next_image = next(images)
|
||||
yield prev_image
|
||||
processed_images += 1
|
||||
pbar.update_absolute(processed_images, total_images)
|
||||
prev_image = next_image
|
||||
except StopIteration:
|
||||
pass
|
||||
if prev_image is not None:
|
||||
yield prev_image
|
||||
|
||||
|
||||
def load_images_list(img_list: list, ):
|
||||
gen = images_generator(img_list)
|
||||
(width, height) = next(gen)
|
||||
images = torch.from_numpy(np.fromiter(gen, np.dtype((np.float32, (height, width, 3)))))
|
||||
if len(images) == 0:
|
||||
raise FileNotFoundError(f"No images could be loaded .")
|
||||
return images
|
||||
|
||||
def get_video_files(directory, extensions=None):
|
||||
if extensions is None:
|
||||
extensions = ['webm', 'mp4', 'mkv', 'gif', 'mov']
|
||||
extensions = [ext.lower() for ext in extensions]
|
||||
video_files = []
|
||||
|
||||
for root, dirs, files in os.walk(directory):
|
||||
for file in files:
|
||||
_, ext = os.path.splitext(file)
|
||||
ext = ext.lower()[1:]
|
||||
if ext in extensions:
|
||||
full_path = os.path.join(root, file)
|
||||
video_files.append(full_path)
|
||||
return video_files
|
||||
+3
-3
@@ -1,9 +1,9 @@
|
||||
[project]
|
||||
name = "grag_image_editing"
|
||||
description = "GRAG-Image-Editing : Group-Relative Attention Guidance for Image Editing,you can try it in comfyUI"
|
||||
version = "1.0.0"
|
||||
description = "GRAG-Image-Editing : Group-Relative Attention Guidance for Image Editing, on the native comfy DiT stack"
|
||||
version = "2.0.0"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = ["accelerate", "diffusers", "ipykernel", "matplotlib", "pathlib", "pickleshare", "pip-chill", "termcolor", "tomli", "torch", "torchvision", "transformers", "huggingface_hub"]
|
||||
dependencies = []
|
||||
|
||||
[project.urls]
|
||||
Repository = "https://github.com/smthemex/ComfyUI_GRAG_Image_Editing"
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
accelerate
|
||||
diffusers
|
||||
ipykernel
|
||||
matplotlib
|
||||
pathlib
|
||||
pickleshare
|
||||
pip-chill
|
||||
termcolor
|
||||
tomli
|
||||
torch
|
||||
torchvision
|
||||
transformers
|
||||
huggingface_hub
|
||||
Reference in New Issue
Block a user