Junst
cfadc10246
🔧 **Flux Dual CLIP Integration**:
1. **Root Cause Analysis**:
- CLIP errors: Wrong CLIP structure (single vs dual)
- Attention issues: Failed text encoding due to wrong CLIP
- Concept processing failure: No valid embeddings
2. **Dual CLIP Support**:
- Updated workflow to use DualCLIPLoader
- Proper Flux CLIP structure (CLIP-L + T5-XXL)
- Enhanced text encoding for dual CLIP architecture
3. **Complete Fix**:
- DiffusionModelLoader: flux1-dev.safetensors
- DualCLIPLoader: clip_l + t5xxl for proper dual CLIP
- ConceptDiffusion now supports Flux dual CLIP structure
This should resolve both CLIP errors and attention capture issues by using the correct Flux architecture.
199 lines
5.9 KiB
JSON
199 lines
5.9 KiB
JSON
{
|
|
"last_node_id": 11,
|
|
"last_link_id": 11,
|
|
"nodes": [
|
|
{
|
|
"id": 1,
|
|
"type": "LoadImage",
|
|
"pos": [100, 100],
|
|
"size": {"0": 315, "1": 314},
|
|
"flags": {},
|
|
"order": 0,
|
|
"mode": 0,
|
|
"outputs": [
|
|
{"name": "IMAGE", "type": "IMAGE", "links": [1, 7], "slot_index": 0}
|
|
],
|
|
"properties": {"Node name for S&R": "LoadImage"},
|
|
"widgets_values": ["example_image.png", "image"]
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "DiffusionModelLoader",
|
|
"pos": [100, 500],
|
|
"size": {"0": 315, "1": 98},
|
|
"flags": {},
|
|
"order": 1,
|
|
"mode": 0,
|
|
"outputs": [
|
|
{"name": "MODEL", "type": "MODEL", "links": [2], "slot_index": 0}
|
|
],
|
|
"properties": {"Node name for S&R": "DiffusionModelLoader"},
|
|
"widgets_values": ["flux1-dev.safetensors"]
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "DualCLIPLoader",
|
|
"pos": [100, 650],
|
|
"size": {"0": 315, "1": 98},
|
|
"flags": {},
|
|
"order": 2,
|
|
"mode": 0,
|
|
"outputs": [
|
|
{"name": "CLIP", "type": "CLIP", "links": [3], "slot_index": 0}
|
|
],
|
|
"properties": {"Node name for S&R": "DualCLIPLoader"},
|
|
"widgets_values": ["clip_l.safetensors", "t5xxl_fp16.safetensors", "flux"]
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "ConceptAttentionNode",
|
|
"pos": [500, 100],
|
|
"size": {"0": 400, "1": 400},
|
|
"flags": {},
|
|
"order": 2,
|
|
"mode": 0,
|
|
"inputs": [
|
|
{"name": "model", "type": "MODEL", "link": 2},
|
|
{"name": "clip", "type": "CLIP", "link": 3},
|
|
{"name": "image", "type": "IMAGE", "link": 1}
|
|
],
|
|
"outputs": [
|
|
{"name": "concept_maps", "type": "CONCEPT_MAPS", "links": [4, 5], "slot_index": 0},
|
|
{"name": "visualized_image", "type": "IMAGE", "links": [6], "slot_index": 1}
|
|
],
|
|
"properties": {"Node name for S&R": "ConceptAttentionNode"},
|
|
"widgets_values": ["Elderly woman character with white hair and a red dress holding a cane and accompanied by a small white cat on her shoulder. Cartoon style with smooth lines and exaggerated facial features offering a whimsical and friendly appearance. Her name is abae.", 20, 0]
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "ConceptSaliencyMapNode",
|
|
"pos": [1000, 100],
|
|
"size": {"0": 300, "1": 200},
|
|
"flags": {},
|
|
"order": 3,
|
|
"mode": 0,
|
|
"inputs": [
|
|
{"name": "concept_maps", "type": "CONCEPT_MAPS", "link": 4},
|
|
{"name": "concept_name", "type": "STRING", "link": null},
|
|
{"name": "threshold", "type": "FLOAT", "link": null}
|
|
],
|
|
"outputs": [
|
|
{"name": "mask", "type": "MASK", "links": null, "slot_index": 0},
|
|
{"name": "saliency_image", "type": "IMAGE", "links": [8], "slot_index": 1}
|
|
],
|
|
"properties": {"Node name for S&R": "ConceptSaliencyMapNode"},
|
|
"widgets_values": ["woman", 0.5]
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "ConceptSegmentationNode",
|
|
"pos": [1000, 400],
|
|
"size": {"0": 300, "1": 200},
|
|
"flags": {},
|
|
"order": 4,
|
|
"mode": 0,
|
|
"inputs": [
|
|
{"name": "concept_maps", "type": "CONCEPT_MAPS", "link": 5},
|
|
{"name": "image", "type": "IMAGE", "link": 7},
|
|
{"name": "concepts", "type": "STRING", "link": null}
|
|
],
|
|
"outputs": [
|
|
{"name": "segmentation_mask", "type": "MASK", "links": null, "slot_index": 0},
|
|
{"name": "segmented_image", "type": "IMAGE", "links": [10], "slot_index": 1}
|
|
],
|
|
"properties": {"Node name for S&R": "ConceptSegmentationNode"},
|
|
"widgets_values": ["woman, cat, white, lines, cane"]
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "ConceptAttentionVisualizerNode",
|
|
"pos": [500, 600],
|
|
"size": {"0": 300, "1": 200},
|
|
"flags": {},
|
|
"order": 5,
|
|
"mode": 0,
|
|
"inputs": [
|
|
{"name": "concept_maps", "type": "CONCEPT_MAPS", "link": 4},
|
|
{"name": "image", "type": "IMAGE", "link": 1}
|
|
],
|
|
"outputs": [
|
|
{"name": "visualized_image", "type": "IMAGE", "links": [11], "slot_index": 0}
|
|
],
|
|
"properties": {"Node name for S&R": "ConceptAttentionVisualizerNode"},
|
|
"widgets_values": []
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "SaveImage",
|
|
"pos": [1400, 100],
|
|
"size": {"0": 315, "1": 270},
|
|
"flags": {},
|
|
"order": 6,
|
|
"mode": 0,
|
|
"inputs": [
|
|
{"name": "images", "type": "IMAGE", "link": 6}
|
|
],
|
|
"properties": {"Node name for S&R": "SaveImage"},
|
|
"widgets_values": ["concept_attention_visualization"]
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "SaveImage",
|
|
"pos": [1400, 400],
|
|
"size": {"0": 315, "1": 270},
|
|
"flags": {},
|
|
"order": 7,
|
|
"mode": 0,
|
|
"inputs": [
|
|
{"name": "images", "type": "IMAGE", "link": 8}
|
|
],
|
|
"properties": {"Node name for S&R": "SaveImage"},
|
|
"widgets_values": ["person_saliency_map"]
|
|
},
|
|
{
|
|
"id": 9,
|
|
"type": "SaveImage",
|
|
"pos": [1400, 700],
|
|
"size": {"0": 315, "1": 270},
|
|
"flags": {},
|
|
"order": 8,
|
|
"mode": 0,
|
|
"inputs": [
|
|
{"name": "images", "type": "IMAGE", "link": 10}
|
|
],
|
|
"properties": {"Node name for S&R": "SaveImage"},
|
|
"widgets_values": ["concept_segmentation"]
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "SaveImage",
|
|
"pos": [900, 900],
|
|
"size": {"0": 315, "1": 270},
|
|
"flags": {},
|
|
"order": 9,
|
|
"mode": 0,
|
|
"inputs": [
|
|
{"name": "images", "type": "IMAGE", "link": 11}
|
|
],
|
|
"properties": {"Node name for S&R": "SaveImage"},
|
|
"widgets_values": ["attention_overlay"]
|
|
}
|
|
],
|
|
"links": [
|
|
[1, 1, 0, 3, 2, "IMAGE"],
|
|
[2, 2, 0, 3, 0, "MODEL"],
|
|
[3, 2, 1, 3, 1, "CLIP"],
|
|
[4, 3, 0, 4, 0, "CONCEPT_MAPS"],
|
|
[5, 3, 0, 5, 0, "CONCEPT_MAPS"],
|
|
[6, 3, 1, 7, 0, "IMAGE"],
|
|
[7, 1, 0, 5, 1, "IMAGE"],
|
|
[8, 4, 1, 8, 0, "IMAGE"],
|
|
[10, 5, 1, 9, 0, "IMAGE"],
|
|
[11, 6, 0, 10, 0, "IMAGE"]
|
|
],
|
|
"groups": [],
|
|
"config": {},
|
|
"extra": {},
|
|
"version": 0.4
|
|
}
|