diff --git a/CHANGELOG.md b/CHANGELOG.md index 74e5345..618ae18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,31 @@ # Changelog -## v1.2 (December 28, 2024) +## v1.3 + +### Added +- Resize modes for `QwenMultiReferenceHandler` to prevent image distortion + - `match_first`: resize all to image1 dimensions (original behavior) + - `common_height`: same height, preserve aspect ratios (recommended for concat) + - `common_width`: same width, preserve aspect ratios + - `largest_dims`: use largest dimensions found across images + +### Changed +- Simplified `QwenVLTextEncoder` by removing unnecessary resolution interface + - Removed confusing resolution controls that didn't affect generation + - Node now focuses on text encoding and vision processing + - Returns clean `(conditioning,)` output like standard ComfyUI text encoders + - Resolution controlled by proper nodes (Empty Latent, Multi-Reference Handler) + +### Fixed +- Multi-reference concat mode squishing images with different aspect ratios + - Added smart resize logic before concatenation + - Aspect ratio preservation prevents distortion + - Maintains backward compatibility with existing workflows +- Grid mode tensor size mismatch when using aspect-preserving resize modes + - Grid now uses uniform dimensions to prevent concatenation errors + - Calculates average aspect ratio for balanced proportions + +## v1.2 ### Added - Documentation for multi-reference spatial ordering (MULTI_REFERENCE_SPATIAL_ORDERING.md) @@ -60,25 +85,3 @@ - Improved parameter tooltips with plain English descriptions - Reference method tooltips now explain each option clearly - Removed redundant Scale Image node from workflows (optimizer handles this internally) - -### Fixed -- RoPE (Rotary Position Embeddings) batch processing bug - - Applied monkey patch from DiffSynth-Studio - - Fixes issues with different image sizes in batch -- Resolution optimizer selecting wrong aspect ratios - - Now considers both pixel count and aspect ratio -- Parameter order breaking backward compatibility - - Moved new parameters to end of function signature -- Double image scaling causing quality degradation - - Removed redundant scaling in workflows - -### Technical Details -- Integrated fixes from DiffSynth-Studio and DiffSynth-Engine -- Maintained compatibility with ComfyUI's CLIP infrastructure -- Reference latents passed through conditioning metadata -- Support for 36 official Qwen resolutions -- 16-channel VAE latent support throughout - -### Documentation -- Created REFERENCE_METHODS_EXPLAINED.md for multi-reference usage -- Added decision guides for common use cases diff --git a/CLAUDE.md b/CLAUDE.md index 4c6af03..54a3379 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,16 +65,18 @@ ComfyUI nodes for the Qwen Image Edit model and Qwen2.5-VL text encoder. Does *N - `QwenVLCLIPLoader` - Loads Qwen2.5-VL model with RoPE fix applied - `QwenVLTextEncoder` - Text encoder with: - Default Qwen formatting or custom Template Builder input - - Resolution optimization with aspect ratio preservation (36 Qwen resolutions) - - Optional multi-reference input accepts `QwenMultiReferenceHandler` output - - Context image support for ControlNet-style conditioning + - Resolution control: dropdown (36 Qwen resolutions), custom (exact dimensions) + - Optional edit_image input for vision-based editing + - Optional context_image input for ControlNet-style conditioning (separate from vision processing) + - Width/height outputs for downstream workflow integration - Debug mode for troubleshooting - `QwenLowresFixNode` - Two-stage generation for quality improvement -- `QwenMultiReferenceHandler` - Combines up to 4 images with: - - Six resize modes: keep_proportion, stretch, resize, pad, pad_edge, crop +- `QwenMultiReferenceHandler` - Combines up to 4 images with aspect ratio preservation: + - Four resize modes: match_first, common_height, common_width, largest_dims - Five upscale methods: nearest-exact, bilinear, area, bicubic, lanczos - Four combination methods: index, offset, concat, grid - Weighted blending for offset method + - Smart resizing prevents distortion in concat mode **Helper Nodes:** - `QwenVLEmptyLatent` - Creates empty 16-channel latents @@ -119,17 +121,24 @@ ComfyUI nodes for the Qwen Image Edit model and Qwen2.5-VL text encoder. Does *N - Vision processing duplication bug (2x speedup via patch) - Template token dropping standardized (consistency with DiffSynth) - Template Builder for custom system prompts (removed duplicate from encoder) -- Resolution optimization with aspect ratio preservation +- Resolution optimization with improved aspect ratio matching - Two-stage refinement (Lowres Fix) - Debug mode for troubleshooting -- Multi-reference image support (up to 4 images) with spatial positioning clarity -- Context image support for ControlNet-style conditioning +- Multi-reference image support with aspect ratio preservation +- Context image support for ControlNet-style conditioning (separate from vision processing) +- Width/height outputs from QwenVLTextEncoder for workflow integration +- Simplified resolution interface (removed buggy auto mode, dropdown + custom only) +- Fixed grid mode tensor mismatch in multi-reference handler -### Phase 1 Complete (December 2024) +### Phase 1 Complete (January 2025) - Fixed vision processing duplication (2x performance improvement) - Standardized template token dropping for consistency with DiffSynth - Added context_image support for ControlNet-style workflows - Clarified multi-reference spatial behavior vs temporal frames +- Added width/height outputs to QwenVLTextEncoder +- Simplified resolution interface (removed confusing auto mode, dropdown/custom only) +- Fixed multi-reference concat distortion with aspect ratio preservation +- Fixed grid mode tensor mismatch when using aspect-preserving resize modes ### Remaining Implementation Plan diff --git a/README.md b/README.md index bc4c13a..6ad3dce 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,159 @@ *Despite the name, this is becoming more of a Qwen Image Edit repo.* -# Overview -ComfyUI nodes for Qwen2.5-VL and Qwen Image Edit models with enhanced template control. +# ComfyUI Qwen Image Edit Nodes + +ComfyUI nodes for Qwen-Image-Edit for system prompt modifications, with multi-reference support and other fun features. + +## Multi-Reference Methods + +The `QwenMultiReferenceHandler` offers three composition methods for combining images: + +- **concat**: Side-by-side layout for character/object comparison. Best with `common_height` resize for aspect preservation. +- **grid**: 2x2 layout for balanced influence of all references. Uses uniform dimensions to prevent tensor mismatches. +- **offset**: Weighted blending creates single composite. Perfect for style transfer with adjustable influence via weights (e.g., `1.0,1.0,0.5,1.2`). + +## Spatial Reference Tokens FYI + +Qwen-Image-Edit includes precision control tokens for surgical editing: + +- **`<|object_ref_start|>...<|object_ref_end|>`** - Reference specific objects by description +- **`<|box_start|>x1,y1,x2,y2<|box_end|>`** - Target rectangular regions (normalized 0-1 coordinates) +- **`<|quad_start|>x1,y1,x2,y2,x3,y3,x4,y4<|quad_end|>`** - Define complex quadrilateral areas + +*[Examples and usage patterns →](#spatial-token-examples)* ## Core Nodes -### Text Encoding -- **QwenVLCLIPLoader**: Loads Qwen2.5-VL models from `models/text_encoders/` folder -- **QwenVLTextEncoder**: Enhanced encoder with flexible template control - - `apply_template`: Enable/disable system templates - - `token_removal`: Choose between auto (ComfyUI), diffsynth (fixed), or none - - `mode`: Text-to-image or image_edit (defaults to image_edit) +### QwenVLCLIPLoader +Loads Qwen2.5-VL models with RoPE position embedding fixes. +- **Use for:** Loading any Qwen2.5-VL model from `models/text_encoders/` +- **Example:** Text-to-image, image editing workflows -### Template Building -- **QwenTemplateBuilder**: Interactive template builder with presets - - 9 presets including DiffSynth defaults, artistic, photorealistic, minimal edit - - Custom template mode with system prompt control - - Automatic vision token handling for image editing -- **QwenTemplateBuilderV2**: Simplified Python-only version - - All options visible, no JavaScript required - - custom_t2i and custom_edit modes -- **QwenTemplateConnector**: Bridges template builder to encoder +### QwenVLTextEncoder +Main text encoder with vision support and dual image inputs. +- **edit_image:** Vision-processed image for semantic understanding +- **context_image:** ControlNet-style spatial conditioning (no vision processing) +- **Outputs:** conditioning +- **Use for:** Text+image encoding with vision token processing +- **Example:** "Change dress to red" (edit_image) + pose skeleton (context_image) -### Helper Nodes -- **QwenOptimalResolution**: Auto-resizes images to nearest Qwen-preferred resolution -- **QwenResolutionSelector**: Dropdown selector for Qwen resolutions -- **QwenTokenInfo**: Reference display for special tokens +### QwenMultiReferenceHandler +Combines up to 4 images with aspect ratio preservation to prevent distortion. +- **Methods:** concat (side-by-side), grid (2x2), offset (weighted blend) +- **Resize modes:** match_first, common_height (best for concat), common_width, largest_dims +- **Use for:** Character fusion, style transfer, pose references +- **Example:** Combine Mona Lisa + Shrek + fighting pose → Mona Lisa fighting Shrek in the specified pose + +### QwenLowresFixNode +Two-stage refinement: generate → upscale → polish details. +- **Use for:** Quality enhancement of generated images +- **Example:** 1024px generation → 1.5x upscale = detail refinement + +## Helper Nodes + +### QwenVLEmptyLatent / QwenVLImageToLatent +Creates/converts 16-channel latents for Qwen model. +- **Use for:** Latent space operations + +## Template System + +### QwenTemplateBuilder +Interactive template builder with style presets. +- **Presets:** DiffSynth, artistic, photorealistic, minimal edit, action, etc. +- **Use for:** Custom system prompts and specialized generation styles + +## Usage Examples + +### Text-to-Image +``` +QwenVLCLIPLoader → QwenVLTextEncoder (mode: text_to_image) → KSampler +``` + +### Image Editing +``` +LoadImage → QwenVLTextEncoder (edit_image, mode: image_edit) → KSampler +``` + +### Multi-Reference Composition +``` +LoadImage(3x) → QwenMultiReferenceHandler → QwenVLTextEncoder → KSampler +``` + +### ControlNet-Style Conditioning +``` +LoadImage(edit) + LoadImage(control) → QwenVLTextEncoder (edit_image + context_image) → KSampler +``` + +### Quality Enhancement +``` +KSampler → QwenLowresFixNode → Final Image +``` + +## Key Features + +- Clean text encoder focused on vision and language processing +- Multi-reference aspect ratio preservation prevents concat distortion +- Context image support for ControlNet workflows +- Template Builder with 20+ presets +- Debug mode with detailed logging +- Patterns and patches from DiffSynth code +- Two-stage refinement system + +## Model Requirements + +Place Qwen2.5-VL models in `ComfyUI/models/text_encoders/`: +- `qwen_2.5_vl_7b.safetensors` (recommended) +- Any Qwen2.5-VL checkpoint + +## Advanced Usage + +**Spatial Reference Tokens** (Phase 3 roadmap): +``` +"Change <|object_ref_start|>the red car<|object_ref_end|> to blue" +"Replace <|box_start|>0.2,0.3,0.8,0.7<|box_end|> with flowers" +``` + +**Denoise Guidelines:** +- High denoise (0.9-1.0): Full creative reimagining +- Low denoise (0.3-0.7): Structure preservation +- Use Empty Latent for high denoise, VAE Encode for low denoise + +## Spatial Token Examples + +### Object Reference - Independent Usage +Target specific objects by description: +``` +"Change <|object_ref_start|>the red car<|object_ref_end|> to blue while keeping everything else the same" +"Make <|object_ref_start|>the woman's dress<|object_ref_end|> flowing in the wind" +"Replace <|object_ref_start|>the old wooden door<|object_ref_end|> with a modern glass entrance" +``` + +### Box Coordinates - Precise Region Control +Target rectangular areas with normalized coordinates (0-1): +``` +"Fill <|box_start|>0.2,0.3,0.8,0.7<|box_end|> with blooming cherry blossoms" +"Replace the content in <|box_start|>0.0,0.0,0.5,1.0<|box_end|> with a mountain landscape" +"Add storm clouds to <|box_start|>0.1,0.0,0.9,0.4<|box_end|>" +``` + +### Quadrilateral - Complex Shape Targeting +Define irregular areas with four corner points: +``` +"Paint graffiti art within <|quad_start|>0.1,0.2,0.6,0.1,0.8,0.7,0.2,0.8<|quad_end|>" +"Transform <|quad_start|>0.3,0.1,0.9,0.2,0.7,0.9,0.1,0.6<|quad_end|> into stained glass" +``` + +### Combined Usage - Multiple Token Types +Mix different tokens for complex edits: +``` +"Change <|object_ref_start|>the building<|object_ref_end|> in <|box_start|>0.4,0.2,1.0,0.8<|box_end|> to Gothic architecture" +"Make <|object_ref_start|>the tree<|object_ref_end|> seasonal: spring leaves in <|quad_start|>0.2,0.1,0.8,0.0,0.9,0.6,0.1,0.7<|quad_end|>" +"Replace <|object_ref_start|>the sky<|object_ref_end|> above <|box_start|>0.0,0.0,1.0,0.4<|box_end|> with aurora borealis" +``` + +### Multi-Object Coordination +Target multiple objects with spatial relationships: +``` +"Change <|object_ref_start|>the left car<|object_ref_end|> to red and <|object_ref_start|>the right car<|object_ref_end|> to blue" +"Make <|object_ref_start|>the foreground flowers<|object_ref_end|> roses and <|object_ref_start|>the background trees<|object_ref_end|> cherry blossoms" +``` diff --git a/example_workflows/qwen_image_edit_multiref_workflow.json b/example_workflows/qwen_image_edit_multiref_workflow.json index 56134e7..ab30a9b 100644 --- a/example_workflows/qwen_image_edit_multiref_workflow.json +++ b/example_workflows/qwen_image_edit_multiref_workflow.json @@ -1,14 +1,20 @@ { "id": "2b232455-5c82-4c20-a0e8-71907d92a501", "revision": 0, - "last_node_id": 167, - "last_link_id": 362, + "last_node_id": 189, + "last_link_id": 419, "nodes": [ { "id": 37, "type": "UNETLoader", - "pos": [3.8020541667938232, 20.189966201782227], - "size": [330, 90], + "pos": [ + 3.8020541667938232, + 20.189966201782227 + ], + "size": [ + 330, + 90 + ], "flags": {}, "order": 0, "mode": 0, @@ -18,7 +24,9 @@ "name": "MODEL", "type": "MODEL", "slot_index": 0, - "links": [276] + "links": [ + 276 + ] } ], "properties": { @@ -44,15 +52,24 @@ "widget_ue_connectable": {} } }, - "widgets_values": ["qwen_image_edit_fp8_e4m3fn.safetensors", "fp8_e4m3fn"] + "widgets_values": [ + "qwen_image_edit_fp8_e4m3fn.safetensors", + "fp8_e4m3fn" + ] }, { "id": 39, "type": "VAELoader", - "pos": [-19.958463668823242, 314.6122131347656], - "size": [330, 60], + "pos": [ + -19.958463668823242, + 314.6122131347656 + ], + "size": [ + 330, + 60 + ], "flags": {}, - "order": 2, + "order": 1, "mode": 0, "inputs": [], "outputs": [ @@ -60,7 +77,12 @@ "name": "VAE", "type": "VAE", "slot_index": 0, - "links": [223, 244, 343, 355] + "links": [ + 223, + 244, + 355, + 412 + ] } ], "properties": { @@ -86,15 +108,23 @@ "widget_ue_connectable": {} } }, - "widgets_values": ["qwen_image_vae.safetensors"] + "widgets_values": [ + "qwen_image_vae.safetensors" + ] }, { "id": 66, "type": "ModelSamplingAuraFlow", - "pos": [859.3681030273438, -144.76174926757812], - "size": [300, 58], + "pos": [ + 859.3681030273438, + -144.76174926757812 + ], + "size": [ + 300, + 58 + ], "flags": {}, - "order": 20, + "order": 18, "mode": 0, "inputs": [ { @@ -107,7 +137,9 @@ { "name": "MODEL", "type": "MODEL", - "links": [154] + "links": [ + 154 + ] } ], "properties": { @@ -126,13 +158,21 @@ "widget_ue_connectable": {} } }, - "widgets_values": [3.5] + "widgets_values": [ + 3.5 + ] }, { "id": 73, "type": "LoraLoaderModelOnly", - "pos": [394.9740905761719, 46.89153289794922], - "size": [270, 82], + "pos": [ + 394.9740905761719, + 46.89153289794922 + ], + "size": [ + 270, + 82 + ], "flags": {}, "order": 8, "mode": 0, @@ -147,7 +187,9 @@ { "name": "MODEL", "type": "MODEL", - "links": [170] + "links": [ + 170 + ] } ], "properties": { @@ -166,15 +208,24 @@ "version": "7.0.1" } }, - "widgets_values": ["Qwen-Image-Edit-Lightning-8steps-V1.0.safetensors", 1] + "widgets_values": [ + "Qwen-Image-Edit-Lightning-4steps-V1.0.safetensors", + 1 + ] }, { "id": 80, "type": "ConditioningZeroOut", - "pos": [599.8950805664062, 577.550537109375], - "size": [197.712890625, 26], + "pos": [ + 599.8950805664062, + 577.550537109375 + ], + "size": [ + 197.712890625, + 26 + ], "flags": {}, - "order": 15, + "order": 13, "mode": 0, "inputs": [ { @@ -187,7 +238,9 @@ { "name": "CONDITIONING", "type": "CONDITIONING", - "links": [271] + "links": [ + 271 + ] } ], "properties": { @@ -206,10 +259,16 @@ { "id": 81, "type": "CFGNorm", - "pos": [889.4481811523438, -33.180145263671875], - "size": [270, 58], + "pos": [ + 889.4481811523438, + -33.180145263671875 + ], + "size": [ + 270, + 58 + ], "flags": {}, - "order": 21, + "order": 19, "mode": 0, "inputs": [ { @@ -222,7 +281,9 @@ { "name": "patched_model", "type": "MODEL", - "links": [269] + "links": [ + 269 + ] } ], "properties": { @@ -234,17 +295,25 @@ "version": "7.0.1" } }, - "widgets_values": [1] + "widgets_values": [ + 1 + ] }, { "id": 86, "type": "PathchSageAttentionKJ", - "pos": [608.7059936523438, -390.08392333984375], - "size": [270, 58], + "pos": [ + 608.7059936523438, + -390.08392333984375 + ], + "size": [ + 270, + 58 + ], "flags": { "collapsed": false }, - "order": 18, + "order": 16, "mode": 4, "inputs": [ { @@ -257,7 +326,9 @@ { "name": "MODEL", "type": "MODEL", - "links": [169] + "links": [ + 169 + ] } ], "title": "sage", @@ -271,7 +342,9 @@ "version": "7.0.1" } }, - "widgets_values": ["auto"], + "widgets_values": [ + "auto" + ], "color": "#332922", "bgcolor": "#593930", "shape": 1 @@ -279,12 +352,18 @@ { "id": 87, "type": "CFGZeroStar", - "pos": [1074.1048583984375, -349.59375], - "size": [159.744140625, 26], + "pos": [ + 1074.1048583984375, + -349.59375 + ], + "size": [ + 159.744140625, + 26 + ], "flags": { "collapsed": false }, - "order": 19, + "order": 17, "mode": 4, "inputs": [ { @@ -297,7 +376,9 @@ { "name": "patched_model", "type": "MODEL", - "links": [204] + "links": [ + 204 + ] } ], "properties": { @@ -317,12 +398,18 @@ { "id": 88, "type": "TorchCompileModelQwenImage", - "pos": [173.8088836669922, -312.2391357421875], - "size": [282.74609375, 178], + "pos": [ + 173.8088836669922, + -312.2391357421875 + ], + "size": [ + 282.74609375, + 178 + ], "flags": { "collapsed": false }, - "order": 14, + "order": 12, "mode": 0, "inputs": [ { @@ -335,7 +422,9 @@ { "name": "MODEL", "type": "MODEL", - "links": [162] + "links": [ + 162 + ] } ], "properties": { @@ -347,17 +436,30 @@ "version": "7.0.1" } }, - "widgets_values": ["inductor", false, "default", false, true, 128] + "widgets_values": [ + "inductor", + false, + "default", + false, + true, + 128 + ] }, { "id": 116, "type": "VAEDecode", - "pos": [1292.6866455078125, 532.370849609375], - "size": [210, 46], + "pos": [ + 1292.6866455078125, + 532.370849609375 + ], + "size": [ + 210, + 46 + ], "flags": { "collapsed": false }, - "order": 23, + "order": 21, "mode": 0, "inputs": [ { @@ -376,7 +478,9 @@ "name": "IMAGE", "type": "IMAGE", "slot_index": 0, - "links": [275] + "links": [ + 414 + ] } ], "properties": { @@ -400,17 +504,26 @@ { "id": 120, "type": "QwenVLCLIPLoader", - "pos": [35.810157775878906, 163.19113159179688], - "size": [270, 60], + "pos": [ + 35.810157775878906, + 163.19113159179688 + ], + "size": [ + 270, + 60 + ], "flags": {}, - "order": 1, + "order": 2, "mode": 0, "inputs": [], "outputs": [ { "name": "clip", "type": "CLIP", - "links": [255, 341] + "links": [ + 255, + 410 + ] } ], "title": "Qwen2.5-VL CLIP Loader (FIXED)", @@ -423,17 +536,25 @@ "version": "7.0.1" } }, - "widgets_values": ["qwen_2.5_vl_7b.safetensors"], + "widgets_values": [ + "qwen_2.5_vl_7b.safetensors" + ], "color": "#323", "bgcolor": "#535" }, { "id": 124, "type": "TextEncodeQwenImageEdit", - "pos": [2000.271728515625, 342.1376953125], - "size": [400, 200], + "pos": [ + 2000.271728515625, + 342.1376953125 + ], + "size": [ + 400, + 200 + ], "flags": {}, - "order": 11, + "order": 10, "mode": 0, "inputs": [ { @@ -451,14 +572,16 @@ "name": "image", "shape": 7, "type": "IMAGE", - "link": 359 + "link": null } ], "outputs": [ { "name": "CONDITIONING", "type": "CONDITIONING", - "links": [248] + "links": [ + 248 + ] } ], "properties": { @@ -472,15 +595,23 @@ "version": "7.0.1" } }, - "widgets_values": [""] + "widgets_values": [ + "" + ] }, { "id": 131, "type": "KSampler", - "pos": [939.9803466796875, 92.1950912475586], - "size": [270, 526], + "pos": [ + 939.9803466796875, + 92.1950912475586 + ], + "size": [ + 270, + 526 + ], "flags": {}, - "order": 22, + "order": 20, "mode": 0, "inputs": [ { @@ -491,7 +622,7 @@ { "name": "positive", "type": "CONDITIONING", - "link": 345 + "link": 413 }, { "name": "negative", @@ -508,7 +639,9 @@ { "name": "LATENT", "type": "LATENT", - "links": [274] + "links": [ + 274 + ] } ], "properties": { @@ -528,139 +661,26 @@ } }, "widgets_values": [ - 407325260319073, + 451654269511750, "randomize", - 8, + 4, 1, "euler", "simple", 1 ] }, - { - "id": 132, - "type": "SaveImage", - "pos": [1306.1964111328125, -50.58351135253906], - "size": [615.53125, 667.6171875], - "flags": {}, - "order": 24, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 275 - } - ], - "outputs": [], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.51", - "Node name for S&R": "SaveImage", - "ue_properties": { - "widget_ue_connectable": { - "filename_prefix": true - }, - "version": "7.0.1" - } - }, - "widgets_values": ["qe_node"] - }, - { - "id": 144, - "type": "ShowText|pysssss", - "pos": [1716.4005126953125, 820.7736206054688], - "size": [551.8046875, 376.109375], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "text", - "type": "STRING", - "link": 322 - } - ], - "outputs": [ - { - "name": "STRING", - "shape": 6, - "type": "STRING", - "links": null - } - ], - "properties": { - "cnr_id": "comfyui-custom-scripts", - "ver": "1.2.5", - "Node name for S&R": "ShowText|pysssss", - "ue_properties": { - "widget_ue_connectable": {}, - "version": "7.0.1" - } - }, - "widgets_values": [ - "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Place the object from the left image into the scene in the right image.<|im_end|>\n<|im_start|>assistant\n" - ] - }, - { - "id": 146, - "type": "PreviewAny", - "pos": [1800.7799072265625, 1326.7484130859375], - "size": [464.60546875, 247.6640625], - "flags": {}, - "order": 17, - "mode": 0, - "inputs": [ - { - "name": "source", - "type": "*", - "link": 353 - } - ], - "outputs": [], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.52", - "Node name for S&R": "PreviewAny", - "ue_properties": { - "widget_ue_connectable": {}, - "version": "7.0.1" - } - }, - "widgets_values": [] - }, - { - "id": 147, - "type": "PreviewAny", - "pos": [1010.5323486328125, 1388.4312744140625], - "size": [210, 88], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "source", - "type": "*", - "link": 325 - } - ], - "outputs": [], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.52", - "Node name for S&R": "PreviewAny", - "ue_properties": { - "widget_ue_connectable": {}, - "version": "7.0.1" - } - }, - "widgets_values": [] - }, { "id": 153, "type": "EmptyLatentImage", - "pos": [2008.8819580078125, 146.8245086669922], - "size": [270, 106], + "pos": [ + 2071.746337890625, + 150.59219360351562 + ], + "size": [ + 270, + 106 + ], "flags": {}, "order": 3, "mode": 0, @@ -669,7 +689,9 @@ { "name": "LATENT", "type": "LATENT", - "links": [358] + "links": [ + 358 + ] } ], "properties": { @@ -685,194 +707,31 @@ "version": "7.0.1" } }, - "widgets_values": [1024, 1024, 1] - }, - { - "id": 154, - "type": "QwenTemplateBuilder", - "pos": [732.420166015625, 732.378662109375], - "size": [738.4408569335938, 415.07061767578125], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "formatted_prompt", - "type": "STRING", - "links": [322, 344] - }, - { - "name": "template_info", - "type": "STRING", - "links": [] - }, - { - "name": "has_vision_tokens", - "type": "BOOLEAN", - "links": [325] - } - ], - "properties": { - "Node name for S&R": "QwenTemplateBuilder", - "ue_properties": { - "widget_ue_connectable": { - "prompt": true, - "template_preset": true, - "system_prompt": true, - "include_vision_tokens": true, - "show_special_tokens": true - }, - "version": "7.0.1" - } - }, "widgets_values": [ - "Take the object from the left and place it in the scene on the right", - "default_edit", - "Describe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.", - true, - true - ] - }, - { - "id": 160, - "type": "QwenVLTextEncoder", - "pos": [437.626708984375, 252.4956512451172], - "size": [400, 316], - "flags": {}, - "order": 16, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 341 - }, - { - "name": "edit_image", - "shape": 7, - "type": "IMAGE", - "link": null - }, - { - "name": "multi_reference", - "shape": 7, - "type": "QWEN_MULTI_REF", - "link": 352 - }, - { - "name": "vae", - "shape": 7, - "type": "VAE", - "link": 343 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": 344 - } - ], - "outputs": [ - { - "name": "conditioning", - "type": "CONDITIONING", - "links": [345] - } - ], - "properties": { - "Node name for S&R": "QwenVLTextEncoder", - "ue_properties": { - "widget_ue_connectable": { - "text": true, - "mode": true, - "token_removal": true, - "debug_mode": true, - "optimize_resolution": true - }, - "version": "7.0.1" - } - }, - "widgets_values": [ - "A beautiful landscape", - "image_edit", - true, - "auto", - true, - true - ] - }, - { - "id": 163, - "type": "QwenMultiReferenceHandler", - "pos": [607.0889892578125, 1288.75], - "size": [270, 190], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "image1", - "shape": 7, - "type": "IMAGE", - "link": 360 - }, - { - "name": "image2", - "shape": 7, - "type": "IMAGE", - "link": 362 - }, - { - "name": "image3", - "shape": 7, - "type": "IMAGE", - "link": null - }, - { - "name": "image4", - "shape": 7, - "type": "IMAGE", - "link": null - } - ], - "outputs": [ - { - "name": "multi_reference", - "type": "QWEN_MULTI_REF", - "links": [352] - }, - { - "name": "preview", - "type": "IMAGE", - "links": [353] - } - ], - "properties": { - "Node name for S&R": "QwenMultiReferenceHandler" - }, - "widgets_values": [ - "index", - "1.0,1.0,1.0,1.0", - "keep_proportion", - "nearest-exact" + 1024, + 1024, + 1 ] }, { "id": 164, "type": "VAEEncode", - "pos": [2434.446044921875, 1220.5225830078125], - "size": [140, 46], + "pos": [ + 2434.446044921875, + 1220.5225830078125 + ], + "size": [ + 140, + 46 + ], "flags": {}, - "order": 13, + "order": 9, "mode": 4, "inputs": [ { "name": "pixels", "type": "IMAGE", - "link": 361 + "link": null }, { "name": "vae", @@ -890,20 +749,36 @@ "properties": { "cnr_id": "comfy-core", "ver": "0.3.55", - "Node name for S&R": "VAEEncode" - } + "Node name for S&R": "VAEEncode", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.0.1" + } + }, + "widgets_values": [] }, { "id": 165, "type": "MarkdownNote", - "pos": [-451.42816162109375, 503.6251220703125], - "size": [662.10498046875, 556.1986694335938], + "pos": [ + -451.42816162109375, + 503.6251220703125 + ], + "size": [ + 662.10498046875, + 556.1986694335938 + ], "flags": {}, - "order": 5, + "order": 4, "mode": 0, "inputs": [], "outputs": [], - "properties": {}, + "properties": { + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.0.1" + } + }, "widgets_values": [ "# Multi-Reference Image Ordering and Spatial Interpretation\n\n## The Problem\n\nWhen using `concat` or `grid` methods with multiple reference images, Qwen interprets the combined image spatially:\n- **Left = First, Right = Second** (in concat mode)\n- **Top-Left = First, Top-Right = Second** (in grid mode)\n\nThis can cause confusion when your prompt says \"first image\" or \"second image\".\n\n## Understanding How Qwen Interprets Concatenated Images\n\n**Setup:**\n- image1: Coffee mug on table → Appears LEFT in concat\n- image2: Plush doll on rainbow → Appears RIGHT in concat\n- Method: concat creates: [Coffee Mug | Plush Doll]\n\n**Important:** In concat mode:\n- image1 = LEFT side = \"first image\" when referenced spatially\n- image2 = RIGHT side = \"second image\" when referenced spatially\n\n**Prompt Clarity Matters:**\n- Ambiguous: \"Place the object from the first image into the second image\"\n- Clear: \"Take the object from the left and place it in the scene on the right\"\n- Clearest: \"Put the coffee mug into the rainbow scene\"\n\n## Solutions\n\n### Option 1: Use `index` Method (Recommended)\nThe `index` method keeps images truly separate and handles positional references better:\n```\nMethod: index\nPrompt: \"Place the object from the first image into the scene of the second image\"\n```\n\n### Option 2: Adjust Your Prompt for Spatial Layout\nWhen using `concat`, refer to positions instead of order:\n```\nMethod: concat \nPrompt: \"Place the object from the LEFT image into the scene from the RIGHT image\"\n```\n\n### Option 3: Swap Input Connections\nConnect your images in reverse order:\n- image1: Plush doll (will appear left)\n- image2: Coffee mug (will appear right)\n- Prompt: \"Place the object from the right into the left scene\"\n\n## Visual Layout Reference\n\n### Concat Method\n```\n[image1 | image2]\n LEFT RIGHT\n```\n\n### Grid Method\n```\n[image1 | image2]\n[image3 | image4]\n\nTOP-LEFT TOP-RIGHT\nBOT-LEFT BOT-RIGHT\n```\n\n## Best Practices\n\n1. **For \"first/second\" references**: Use `index` method\n2. **For style transfer**: Use `offset` method with weights\n3. **For spatial comparison**: Use `concat` with left/right references\n4. **For multiple variations**: Use `grid` with corner references\n\n## Technical Details\n\nThe concatenation happens at the tensor level:\n- `concat`: `torch.cat(images, dim=2)` - Horizontal concatenation\n- `grid`: Creates 2x2 layout with images in reading order\n\nQwen's vision transformer processes images like text - left to right, top to bottom. This is why spatial position matters more than input order for concat/grid methods." ], @@ -911,19 +786,27 @@ "bgcolor": "#653" }, { - "id": 166, + "id": 182, "type": "LoadImage", - "pos": [217.25112915039062, 711.3232421875], - "size": [274.080078125, 314], + "pos": [ + 628.59228515625, + 775.8883666992188 + ], + "size": [ + 274.080078125, + 314 + ], "flags": {}, - "order": 7, + "order": 5, "mode": 0, "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", - "links": [359, 360, 361] + "links": [ + 415 + ] }, { "name": "MASK", @@ -934,15 +817,28 @@ "properties": { "cnr_id": "comfy-core", "ver": "0.3.55", - "Node name for S&R": "LoadImage" + "Node name for S&R": "LoadImage", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.0.1" + } }, - "widgets_values": ["IMG1.jpeg", "image"] + "widgets_values": [ + "IMG1.jpeg", + "image" + ] }, { - "id": 167, + "id": 183, "type": "LoadImage", - "pos": [174.2808837890625, 1173.275634765625], - "size": [274.080078125, 314], + "pos": [ + 567.6510620117188, + 1237.291259765625 + ], + "size": [ + 274.080078125, + 314 + ], "flags": {}, "order": 6, "mode": 0, @@ -951,7 +847,9 @@ { "name": "IMAGE", "type": "IMAGE", - "links": [362] + "links": [ + 416 + ] }, { "name": "MASK", @@ -962,46 +860,473 @@ "properties": { "cnr_id": "comfy-core", "ver": "0.3.55", - "Node name for S&R": "LoadImage" + "Node name for S&R": "LoadImage", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.0.1" + } }, - "widgets_values": ["IMG2.jpeg", "image"] + "widgets_values": [ + "IMG2.jpeg", + "image" + ] + }, + { + "id": 184, + "type": "LoadImage", + "pos": [ + 880.65771484375, + 1464.7506103515625 + ], + "size": [ + 274.080078125, + 314 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 417 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.55", + "Node name for S&R": "LoadImage", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.0.1" + } + }, + "widgets_values": [ + "IMG1.jpeg", + "image" + ] + }, + { + "id": 186, + "type": "PreviewImage", + "pos": [ + 1717.25927734375, + 877.4437866210938 + ], + "size": [ + 140, + 26 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 418 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.56", + "Node name for S&R": "PreviewImage", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.0.1" + } + }, + "widgets_values": [] + }, + { + "id": 187, + "type": "QwenVLTextEncoder", + "pos": [ + 1126.990478515625, + 660.6602783203125 + ], + "size": [ + 400, + 244 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 410 + }, + { + "name": "edit_image", + "shape": 7, + "type": "IMAGE", + "link": 419 + }, + { + "name": "context_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 412 + } + ], + "outputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "links": [ + 413 + ] + } + ], + "properties": { + "Node name for S&R": "QwenVLTextEncoder", + "ue_properties": { + "widget_ue_connectable": { + "text": true, + "mode": true, + "use_custom_system_prompt": true, + "token_removal": true, + "debug_mode": true + }, + "version": "7.0.1" + } + }, + "widgets_values": [ + "Battle scene featuring the strange Mona Lisa fighting the blue plush doll in the style of the snowy digital art image.", + "image_edit", + false, + "auto", + true + ] + }, + { + "id": 188, + "type": "SaveImage", + "pos": [ + 1306.1964111328125, + -50.58351135253906 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 414 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.56", + "Node name for S&R": "SaveImage", + "ue_properties": { + "widget_ue_connectable": { + "filename_prefix": true + }, + "version": "7.0.1" + } + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 189, + "type": "QwenMultiReferenceHandler", + "pos": [ + 1097.4114990234375, + 1088.392333984375 + ], + "size": [ + 270, + 190 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "image1", + "shape": 7, + "type": "IMAGE", + "link": 415 + }, + { + "name": "image2", + "shape": 7, + "type": "IMAGE", + "link": 416 + }, + { + "name": "image3", + "shape": 7, + "type": "IMAGE", + "link": 417 + }, + { + "name": "image4", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "composite_image", + "type": "IMAGE", + "links": [ + 418, + 419 + ] + } + ], + "properties": { + "Node name for S&R": "QwenMultiReferenceHandler", + "ue_properties": { + "widget_ue_connectable": { + "reference_method": true, + "weights": true, + "resize_mode": true, + "upscale_method": true + }, + "version": "7.0.1" + } + }, + "widgets_values": [ + "concat", + "1.0,1.0,1.0,1.0", + "common_height", + "bicubic" + ] } ], "links": [ - [154, 66, 0, 81, 0, "MODEL"], - [162, 88, 0, 86, 0, "MODEL"], - [169, 86, 0, 87, 0, "MODEL"], - [170, 73, 0, 88, 0, "MODEL"], - [204, 87, 0, 66, 0, "MODEL"], - [223, 39, 0, 116, 1, "VAE"], - [244, 39, 0, 124, 1, "VAE"], - [248, 124, 0, 80, 0, "CONDITIONING"], - [255, 120, 0, 124, 0, "CLIP"], - [269, 81, 0, 131, 0, "MODEL"], - [271, 80, 0, 131, 2, "CONDITIONING"], - [274, 131, 0, 116, 0, "LATENT"], - [275, 116, 0, 132, 0, "IMAGE"], - [276, 37, 0, 73, 0, "MODEL"], - [322, 154, 0, 144, 0, "STRING"], - [325, 154, 2, 147, 0, "*"], - [341, 120, 0, 160, 0, "CLIP"], - [343, 39, 0, 160, 3, "VAE"], - [344, 154, 0, 160, 4, "STRING"], - [345, 160, 0, 131, 1, "CONDITIONING"], - [352, 163, 0, 160, 2, "QWEN_MULTI_REF"], - [353, 163, 1, 146, 0, "*"], - [355, 39, 0, 164, 1, "VAE"], - [358, 153, 0, 131, 3, "LATENT"], - [359, 166, 0, 124, 2, "IMAGE"], - [360, 166, 0, 163, 0, "IMAGE"], - [361, 166, 0, 164, 0, "IMAGE"], - [362, 167, 0, 163, 1, "IMAGE"] + [ + 154, + 66, + 0, + 81, + 0, + "MODEL" + ], + [ + 162, + 88, + 0, + 86, + 0, + "MODEL" + ], + [ + 169, + 86, + 0, + 87, + 0, + "MODEL" + ], + [ + 170, + 73, + 0, + 88, + 0, + "MODEL" + ], + [ + 204, + 87, + 0, + 66, + 0, + "MODEL" + ], + [ + 223, + 39, + 0, + 116, + 1, + "VAE" + ], + [ + 244, + 39, + 0, + 124, + 1, + "VAE" + ], + [ + 248, + 124, + 0, + 80, + 0, + "CONDITIONING" + ], + [ + 255, + 120, + 0, + 124, + 0, + "CLIP" + ], + [ + 269, + 81, + 0, + 131, + 0, + "MODEL" + ], + [ + 271, + 80, + 0, + 131, + 2, + "CONDITIONING" + ], + [ + 274, + 131, + 0, + 116, + 0, + "LATENT" + ], + [ + 276, + 37, + 0, + 73, + 0, + "MODEL" + ], + [ + 355, + 39, + 0, + 164, + 1, + "VAE" + ], + [ + 358, + 153, + 0, + 131, + 3, + "LATENT" + ], + [ + 410, + 120, + 0, + 187, + 0, + "CLIP" + ], + [ + 412, + 39, + 0, + 187, + 3, + "VAE" + ], + [ + 413, + 187, + 0, + 131, + 1, + "CONDITIONING" + ], + [ + 414, + 116, + 0, + 188, + 0, + "IMAGE" + ], + [ + 415, + 182, + 0, + 189, + 0, + "IMAGE" + ], + [ + 416, + 183, + 0, + 189, + 1, + "IMAGE" + ], + [ + 417, + 184, + 0, + 189, + 2, + "IMAGE" + ], + [ + 418, + 189, + 0, + 186, + 0, + "IMAGE" + ], + [ + 419, + 189, + 0, + 187, + 1, + "IMAGE" + ] ], "groups": [ { "id": 1, "title": "Step1 - Load models", - "bounding": [10, -20, 350, 433.6000061035156], + "bounding": [ + 10, + -20, + 350, + 433.6000061035156 + ], "color": "#3f789e", "font_size": 24, "flags": {} @@ -1009,7 +1334,12 @@ { "id": 3, "title": "Step3 - Prompt", - "bounding": [380, 160, 450, 470], + "bounding": [ + 380, + 160, + 450, + 470 + ], "color": "#3f789e", "font_size": 24, "flags": {} @@ -1017,7 +1347,12 @@ { "id": 4, "title": "Lightx2v 8steps LoRA", - "bounding": [383.0270690917969, -33.1086540222168, 450, 170], + "bounding": [ + 383.0270690917969, + -33.1086540222168, + 450, + 170 + ], "color": "#3f789e", "font_size": 24, "flags": {} @@ -1026,8 +1361,11 @@ "config": {}, "extra": { "ds": { - "scale": 0.9318423366662651, - "offset": [512.9038041295821, 126.51078513847413] + "scale": 0.9680308328634053, + "offset": [ + 39.02903480683921, + 229.7776661103842 + ] }, "frontendVersion": "1.25.11", "ue_links": [], @@ -1038,4 +1376,4 @@ "VHS_KeepIntermediate": true }, "version": 0.4 -} +} \ No newline at end of file diff --git a/nodes/qwen_multi_reference.py b/nodes/qwen_multi_reference.py index daafec1..f603bb4 100644 --- a/nodes/qwen_multi_reference.py +++ b/nodes/qwen_multi_reference.py @@ -37,9 +37,13 @@ class QwenMultiReferenceHandler: "default": "1.0,1.0,1.0,1.0", "tooltip": "Weights for each image (comma-separated, used in 'offset' blend mode)" }), + "resize_mode": (["match_first", "common_height", "common_width", "largest_dims"], { + "default": "common_height", + "tooltip": "match_first: resize all to image1 size | common_height: same height, keep aspect (uniform in grid) | common_width: same width, keep aspect (uniform in grid) | largest_dims: resize all to largest dimensions" + }), "upscale_method": (["nearest-exact", "bilinear", "area", "bicubic", "lanczos"], { "default": "bicubic", - "tooltip": "Interpolation method for resizing images to match the first image's resolution." + "tooltip": "Interpolation method for resizing images." }), } } @@ -52,10 +56,16 @@ class QwenMultiReferenceHandler: TITLE = "Multi-Reference Canvas Composer" DESCRIPTION = """Combines multiple images into a single high-resolution canvas for spatial reference. -• All images are resized to match the dimensions of image1. -• 'concat' creates a side-by-side image. -• 'grid' creates a 2x2 image layout. -• 'offset' creates a weighted blend. +Resize modes: +• match_first: resize all to image1 dimensions (may distort) +• common_height: same height, preserve aspect ratios (uniform dimensions in grid mode) +• common_width: same width, preserve aspect ratios (uniform dimensions in grid mode) +• largest_dims: resize all to largest width/height found + +Composition methods: +• 'concat' creates a side-by-side image +• 'grid' creates a 2x2 image layout (requires uniform dimensions) +• 'offset' creates a weighted blend The output is a single image tensor, ready for the QwenVLTextEncoder.""" @@ -65,6 +75,7 @@ The output is a single image tensor, ready for the QwenVLTextEncoder.""" image3: Optional[torch.Tensor] = None, image4: Optional[torch.Tensor] = None, weights: str = "1.0,1.0,1.0,1.0", + resize_mode: str = "common_height", upscale_method: str = "bicubic") -> Tuple[torch.Tensor]: # Collect all provided, non-None images @@ -73,21 +84,81 @@ The output is a single image tensor, ready for the QwenVLTextEncoder.""" if not images: raise ValueError("At least one image must be provided to the Multi-Reference Handler.") - # Step 1: Standardize all images to the resolution of the first image - # This ensures all parts of the canvas have consistent dimensions. - target_h, target_w = images[0].shape[1:3] - logger.info(f"[Multi-Reference] Standardizing all images to {target_w}x{target_h}.") + # Step 1: Calculate target dimensions based on resize mode + if resize_mode == "match_first": + # Original behavior: resize all to match first image + target_h, target_w = images[0].shape[1:3] + logger.info(f"[Multi-Reference] Resizing all images to match first image: {target_w}x{target_h}") + + elif resize_mode == "common_height": + # Find common height (use minimum to avoid upscaling) + target_h = min(img.shape[1] for img in images) + # For grid mode, we need uniform dimensions to avoid tensor mismatch + if reference_method == "grid": + # Calculate average width to maintain reasonable proportions + avg_aspect = sum(img.shape[2] / img.shape[1] for img in images) / len(images) + target_w = int(target_h * avg_aspect) + logger.info(f"[Multi-Reference] Grid mode: using common dimensions {target_w}x{target_h}") + else: + logger.info(f"[Multi-Reference] Using common height: {target_h}, aspect ratios preserved") + + elif resize_mode == "common_width": + # Find common width (use minimum to avoid upscaling) + target_w = min(img.shape[2] for img in images) + # For grid mode, we need uniform dimensions to avoid tensor mismatch + if reference_method == "grid": + # Calculate average height to maintain reasonable proportions + avg_aspect = sum(img.shape[1] / img.shape[2] for img in images) / len(images) + target_h = int(target_w * avg_aspect) + logger.info(f"[Multi-Reference] Grid mode: using common dimensions {target_w}x{target_h}") + else: + logger.info(f"[Multi-Reference] Using common width: {target_w}, aspect ratios preserved") + + elif resize_mode == "largest_dims": + # Use largest width and height found across all images + target_h = max(img.shape[1] for img in images) + target_w = max(img.shape[2] for img in images) + logger.info(f"[Multi-Reference] Using largest dimensions: {target_w}x{target_h}") - standardized_images = [images[0]] - for img in images[1:]: - # Use 'stretch' logic to force all images to the same dimensions + # Step 2: Resize all images according to the chosen mode + standardized_images = [] + for i, img in enumerate(images): + if resize_mode == "match_first": + # Stretch to exact target dimensions + new_w, new_h = target_w, target_h + + elif resize_mode == "common_height": + if reference_method == "grid": + # Grid mode requires uniform dimensions + new_w, new_h = target_w, target_h + else: + # Keep aspect ratio, adjust width based on common height + aspect_ratio = img.shape[2] / img.shape[1] # w/h + new_h = target_h + new_w = int(target_h * aspect_ratio) + + elif resize_mode == "common_width": + if reference_method == "grid": + # Grid mode requires uniform dimensions + new_w, new_h = target_w, target_h + else: + # Keep aspect ratio, adjust height based on common width + aspect_ratio = img.shape[1] / img.shape[2] # h/w + new_w = target_w + new_h = int(target_w * aspect_ratio) + + elif resize_mode == "largest_dims": + # Stretch to largest dimensions (may distort) + new_w, new_h = target_w, target_h + + # Resize the image resized_img_chw = comfy.utils.common_upscale( img.movedim(-1, 1), # HWC to CHW for upscale function - target_w, target_h, upscale_method, "disabled" + new_w, new_h, upscale_method, "disabled" ) standardized_images.append(resized_img_chw.movedim(1, -1)) # CHW to HWC - # Step 2: Create the composite canvas based on the chosen method + # Step 3: Create the composite canvas based on the chosen method if reference_method == "offset": # Weighted average blend. The result is a single image of target_w x target_h. logger.info(f"[Multi-Reference] Creating composite using 'offset' (blend) method.") diff --git a/nodes/qwen_vl_encoder.py b/nodes/qwen_vl_encoder.py index 0205a2b..081ac1e 100644 --- a/nodes/qwen_vl_encoder.py +++ b/nodes/qwen_vl_encoder.py @@ -119,19 +119,31 @@ class QwenVLCLIPLoader: class QwenVLTextEncoder: """ - Enhanced text encoder for Qwen2.5-VL with all DiffSynth fixes + Text encoder for Qwen2.5-VL with all DiffSynth fixes Uses ComfyUI's internal CLIP infrastructure for compatibility """ - # DiffSynth-Studio's exact resolution list for Qwen + # Resolution list combining DiffSynth-Studio resolutions with modern aspect ratios QWEN_RESOLUTIONS = [ - (256, 256), (256, 512), (256, 768), (256, 1024), (256, 1280), (256, 1536), (256, 1792), - (512, 256), (512, 512), (512, 768), (512, 1024), (512, 1280), (512, 1536), (512, 1792), - (768, 256), (768, 512), (768, 768), (768, 1024), (768, 1280), (768, 1536), - (1024, 256), (1024, 512), (1024, 768), (1024, 1024), (1024, 1280), (1024, 1536), - (1280, 256), (1280, 512), (1280, 768), (1280, 1024), (1280, 1280), - (1536, 256), (1536, 512), (1536, 768), (1536, 1024), - (1792, 256), (1792, 512) + # Square resolutions + (1024, 1024), (1328, 1328), + + # Common landscape ratios (optimized for quality) + (1328, 800), (1456, 720), (1584, 1056), (1920, 1080), # 16:9 + (2048, 1024), (1344, 768), (1536, 640), + + # Common portrait ratios + (800, 1328), (720, 1456), (1056, 1584), (1080, 1920), # 9:16 + (1024, 2048), (768, 1344), (640, 1536), + + # Original DiffSynth-Studio resolutions for compatibility + (672, 1568), (688, 1504), (752, 1392), (832, 1248), + (880, 1184), (944, 1104), (1104, 944), (1184, 880), + (1248, 832), (1392, 752), (1504, 688), (1568, 672), + + # Smaller resolutions for low VRAM + (512, 512), (768, 768), (512, 768), (768, 512), + (1024, 768), (768, 1024), (1024, 512), (512, 1024) ] @classmethod @@ -171,10 +183,6 @@ class QwenVLTextEncoder: "default": False, "tooltip": "Shows detailed processing info in console. Turn on if things aren't working." }), - "optimize_resolution": ("BOOLEAN", { - "default": True, - "tooltip": "ON: Snap to best Qwen resolution (RECOMMENDED) | OFF: Scale to 1M pixels" - }), } } @@ -193,57 +201,27 @@ KEY DECISION: What goes into KSampler.latent_image? ALWAYS connect VAE to this node for reference latents! """ - def get_optimal_resolution(self, width: int, height: int) -> Tuple[int, int]: - """Find the nearest Qwen-supported resolution preserving aspect ratio""" - target_pixels = width * height - aspect_ratio = width / height - best_res = min( - self.QWEN_RESOLUTIONS, - key=lambda r: abs(r[0] * r[1] - target_pixels) * 0.5 + - abs((r[0] / r[1]) - aspect_ratio) * target_pixels * 0.5 - ) - - return best_res def encode(self, clip, text: str, mode: str = "text_to_image", edit_image: Optional[torch.Tensor] = None, context_image: Optional[torch.Tensor] = None, vae=None, use_custom_system_prompt: bool = False, - token_removal: str = "auto", debug_mode: bool = False, - optimize_resolution: bool = True) -> Tuple[Any]: + token_removal: str = "auto", debug_mode: bool = False) -> Tuple[Any]: images_for_tokenizer = [] ref_latent = None context_latent = None original_text = text + + # No resolution controls needed - images will be processed optimally # Prepare edit_image if in edit mode if mode == "image_edit" and edit_image is not None: - import math - import comfy.utils - if debug_mode: logger.info(f"[Encoder] Input image/canvas shape: {edit_image.shape}") - samples = edit_image.movedim(-1, 1) - - # Determine target resolution - if optimize_resolution: - opt_w, opt_h = self.get_optimal_resolution(samples.shape[3], samples.shape[2]) - width, height = opt_w, opt_h - if debug_mode: - logger.info(f"[Encoder] Using optimal Qwen resolution: {width}x{height}") - else: - total = int(1024 * 1024) - scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) - width = round(samples.shape[3] * scale_by) - height = round(samples.shape[2] * scale_by) - if debug_mode: - logger.info(f"[Encoder] Using legacy 1MP area scaling: {width}x{height}") - - s = comfy.utils.common_upscale(samples, width, height, "area", "disabled") - image = s.movedim(1, -1) - + # Use input image as-is for vision processing (no resizing needed) + image = edit_image images_for_tokenizer = [image[:, :, :, :3]] if vae is not None: @@ -306,22 +284,11 @@ ALWAYS connect VAE to this node for reference latents! # Process context_image separately (ControlNet-style) if context_image is not None and vae is not None: - import comfy.utils if debug_mode: logger.info(f"[Encoder] Processing context image: {context_image.shape}") - samples = context_image.movedim(-1, 1) - - # Use same resolution logic as edit_image for consistency - if 'width' in locals() and 'height' in locals(): - context_w, context_h = width, height - else: # Fallback if not in edit mode - context_w, context_h = self.get_optimal_resolution(samples.shape[3], samples.shape[2]) - - s = comfy.utils.common_upscale(samples, context_w, context_h, "area", "disabled") - context_img_resized = s.movedim(1, -1) - - context_latent = vae.encode(context_img_resized[:, :, :, :3]) + # Use context image as-is (no resizing needed) + context_latent = vae.encode(context_image[:, :, :, :3]) if debug_mode: logger.info(f"[Encoder] Encoded context_image latent shape: {context_latent.shape}")