diff --git a/examples/flux1dev_2gpu_GGUF.json b/examples/flux1dev_2gpu_gguf.json similarity index 100% rename from examples/flux1dev_2gpu_GGUF.json rename to examples/flux1dev_2gpu_gguf.json diff --git a/examples/flux1dev_cpu_1gpu_GGUF.json b/examples/flux1dev_cpu_1gpu_gguf.json similarity index 100% rename from examples/flux1dev_cpu_1gpu_GGUF.json rename to examples/flux1dev_cpu_1gpu_gguf.json diff --git a/examples/flux1dev_gguf_distorch.json b/examples/flux1dev_gguf_distorch.json new file mode 100644 index 0000000..5bb0145 --- /dev/null +++ b/examples/flux1dev_gguf_distorch.json @@ -0,0 +1,650 @@ +{ + "last_node_id": 52, + "last_link_id": 103, + "nodes": [ + { + "id": 22, + "type": "BasicGuider", + "pos": [ + 383.62896728515625, + 96.27555847167969 + ], + "size": [ + 161.1999969482422, + 46 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 102, + "slot_index": 0 + }, + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 87, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 30 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BasicGuider" + }, + "widgets_values": [] + }, + { + "id": 16, + "type": "KSamplerSelect", + "pos": [ + 398.8528137207031, + 360.1959533691406 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 19 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect" + }, + "widgets_values": [ + "deis" + ] + }, + { + "id": 13, + "type": "SamplerCustomAdvanced", + "pos": [ + 883.27783203125, + 267.0603942871094 + ], + "size": [ + 236.8000030517578, + 387.28668212890625 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "noise", + "type": "NOISE", + "link": 37, + "slot_index": 0 + }, + { + "name": "guider", + "type": "GUIDER", + "link": 30, + "slot_index": 1 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 19, + "slot_index": 2 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 20, + "slot_index": 3 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 82, + "slot_index": 4 + } + ], + "outputs": [ + { + "name": "output", + "type": "LATENT", + "links": [ + 83 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "denoised_output", + "type": "LATENT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "SamplerCustomAdvanced" + }, + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1153.8681640625, + 66.58897399902344 + ], + "size": [ + 512.2255859375, + 692.112548828125 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 85 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "SaveImage" + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 41, + "type": "CLIPTextEncode", + "pos": [ + 420.7973937988281, + 467.3779296875 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 99 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 87 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "A towering technological monolith in a cyberpunk cityscape at night, with \"DisTorch\" emblazoned across its surface in massive neon red letters that illuminate the surrounding buildings. The text occupies the central third of the frame, crafted from glowing plasma tubes and crackling energy. Rain-slicked streets below reflect the brilliant signage, while holographic advertisements and flying vehicles populate the background. Moody atmospheric lighting, heavy contrast, photorealistic textures, cinematic color grading." + ] + }, + { + "id": 35, + "type": "DualCLIPLoaderGGUFMultiGPU", + "pos": [ + -74.29531860351562, + 272.3004455566406 + ], + "size": [ + 435.3804016113281, + 130 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 99 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DualCLIPLoaderGGUFMultiGPU" + }, + "widgets_values": [ + "t5-v1_1-xxl-encoder-Q4_K_M.gguf", + "clip_l.safetensors", + "flux", + "cuda:1" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 33, + "type": "VAELoaderMultiGPU", + "pos": [ + -62.57144546508789, + 460.42584228515625 + ], + "size": [ + 450.23980712890625, + 82 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 84 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VAELoaderMultiGPU" + }, + "widgets_values": [ + "ae.safetensors", + "cuda:1" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 17, + "type": "BasicScheduler", + "pos": [ + 660.5484008789062, + 118.03018188476562 + ], + "size": [ + 210, + 106 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 103, + "slot_index": 0 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 20 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "ddim_uniform", + 30, + 1 + ] + }, + { + "id": 40, + "type": "VAEDecode", + "pos": [ + 875.2288818359375, + 120.00743103027344 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 83 + }, + { + "name": "vae", + "type": "VAE", + "link": 84 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 85 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 37, + "type": "UnetLoaderGGUFDisTorchMultiGPU", + "pos": [ + -71.38706970214844, + 90.67648315429688 + ], + "size": [ + 417.6407775878906, + 124 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 102, + 103 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "UnetLoaderGGUFDisTorchMultiGPU" + }, + "widgets_values": [ + "flux1-dev-Q8_0.gguf", + "cuda:0", + "cuda:0,0.4608;cuda:1,0.0271;cpu,0.0136" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 39, + "type": "EmptySD3LatentImage", + "pos": [ + 644.3074951171875, + 299.38372802734375 + ], + "size": [ + 210, + 106 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 82 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptySD3LatentImage" + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 52, + "type": "Note", + "pos": [ + -577.845458984375, + 64.50007629394531 + ], + "size": [ + 495.7899169921875, + 484.21075439453125 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "How the DisTorch allocation string works", + "properties": {}, + "widgets_values": [ + "The allocation string uses a semicolon (;) to separate device allocations, and each device allocation has two parts separated by a comma:\n\nDevice name (e.g., \"cuda:0\", \"cuda:1\", \"cpu\")\nMemory fraction as a decimal (e.g., 0.025 = 2.5%, 0.05 = 5%, 0.10 = 10%)\n\n\nFor my defaults:\n\nCUDA GPU 0: Will allocate 2.5% of its total available memory\nCUDA GPU 1: Will allocate 5% of its total available memory\nCPU: Will allocate 10% of available system memory\n\n\nDisTorch then:\n\nCalculates how much actual memory this means for each device\nDistributes model layers across devices proportionally based on these allocations\nAssigns layers to minimize memory usage while maintaining model functionality\n\n\n\nSo if you have a model with 100 layers and your GPUs have the following memory:\n\nCUDA:0 has 24GB\nCUDA:1 has 24GB\nSystem has 64GB RAM\n\nThe allocation would be:\n\nCUDA:0: 600MB (2.5% of 24GB)\nCUDA:1: 1.2GB (5% of 24GB)\nCPU: 6.4GB (10% of 64GB)\n\nThis would help manage memory usage across your devices while keeping the model functional. The semicolon-separated format makes it easy to specify different allocations for different devices in a single string." + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 25, + "type": "RandomNoise", + "pos": [ + 390.974853515625, + 209.66781616210938 + ], + "size": [ + 236.60824584960938, + 88.84210968017578 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "NOISE", + "type": "NOISE", + "links": [ + 37 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "RandomNoise" + }, + "widgets_values": [ + 5770521, + "fixed" + ] + }, + { + "id": 34, + "type": "UnetLoaderGGUFMultiGPU", + "pos": [ + -79.20733642578125, + -116.61892700195312 + ], + "size": [ + 417.5072937011719, + 97.88888549804688 + ], + "flags": {}, + "order": 7, + "mode": 4, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "UnetLoaderGGUFMultiGPU" + }, + "widgets_values": [ + "flux1-dev-Q8_0.gguf", + "cuda:0" + ], + "color": "#233", + "bgcolor": "#355" + } + ], + "links": [ + [ + 19, + 16, + 0, + 13, + 2, + "SAMPLER" + ], + [ + 20, + 17, + 0, + 13, + 3, + "SIGMAS" + ], + [ + 30, + 22, + 0, + 13, + 1, + "GUIDER" + ], + [ + 37, + 25, + 0, + 13, + 0, + "NOISE" + ], + [ + 82, + 39, + 0, + 13, + 4, + "LATENT" + ], + [ + 83, + 13, + 0, + 40, + 0, + "LATENT" + ], + [ + 84, + 33, + 0, + 40, + 1, + "VAE" + ], + [ + 85, + 40, + 0, + 9, + 0, + "IMAGE" + ], + [ + 87, + 41, + 0, + 22, + 1, + "CONDITIONING" + ], + [ + 99, + 35, + 0, + 41, + 0, + "CLIP" + ], + [ + 102, + 37, + 0, + 22, + 0, + "MODEL" + ], + [ + 103, + 37, + 0, + 17, + 0, + "MODEL" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.8769226950001245, + "offset": [ + 955.3126283315189, + 498.64012539888034 + ] + }, + "ue_links": [], + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "node_versions": { + "comfy-core": "0.3.12", + "ComfyUI-MultiGPU": "3dbfcc713552b0560a471c10d9b29520e8b1981e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/examples/hunyuan_2gpu_GGUF.json b/examples/hunyuan_2gpu_gguf.json similarity index 100% rename from examples/hunyuan_2gpu_GGUF.json rename to examples/hunyuan_2gpu_gguf.json diff --git a/examples/hunyuan_cpu_1gpu_GGUF.json b/examples/hunyuan_cpu_1gpu_gguf.json similarity index 100% rename from examples/hunyuan_cpu_1gpu_GGUF.json rename to examples/hunyuan_cpu_1gpu_gguf.json diff --git a/examples/hunyuan_gguf_distorch.json b/examples/hunyuan_gguf_distorch.json new file mode 100644 index 0000000..80bc707 --- /dev/null +++ b/examples/hunyuan_gguf_distorch.json @@ -0,0 +1,810 @@ +{ + "last_node_id": 119, + "last_link_id": 155, + "nodes": [ + { + "id": 72, + "type": "FluxGuidance", + "pos": [ + 3012.307373046875, + -267.3423767089844 + ], + "size": [ + 211.60000610351562, + 58 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 83 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 79 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FluxGuidance" + }, + "widgets_values": [ + 6 + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 73, + "type": "ModelSamplingSD3", + "pos": [ + 3030.986328125, + 216.49542236328125 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 141 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 78 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ModelSamplingSD3" + }, + "widgets_values": [ + 7 + ] + }, + { + "id": 74, + "type": "VAEDecodeTiled", + "pos": [ + 3246.459716796875, + 191.86683654785156 + ], + "size": [ + 210, + 150 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 85 + }, + { + "name": "vae", + "type": "VAE", + "link": 99 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecodeTiled" + }, + "widgets_values": [ + 256, + 64, + 64, + 8 + ] + }, + { + "id": 75, + "type": "SamplerCustomAdvanced", + "pos": [ + 3488.497314453125, + -150.8239288330078 + ], + "size": [ + 292.4319763183594, + 479.03521728515625 + ], + "flags": { + "collapsed": false + }, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "noise", + "type": "NOISE", + "link": 87, + "slot_index": 0 + }, + { + "name": "guider", + "type": "GUIDER", + "link": 88, + "slot_index": 1 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 89, + "slot_index": 2 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 90, + "slot_index": 3 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 155, + "slot_index": 4 + } + ], + "outputs": [ + { + "name": "output", + "type": "LATENT", + "links": [ + 85 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "denoised_output", + "type": "LATENT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "SamplerCustomAdvanced" + }, + "widgets_values": [] + }, + { + "id": 66, + "type": "BasicScheduler", + "pos": [ + 3250.771484375, + 41.81864929199219 + ], + "size": [ + 210, + 109.8011474609375 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 142, + "slot_index": 0 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 90 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "simple", + 20, + 1 + ] + }, + { + "id": 69, + "type": "KSamplerSelect", + "pos": [ + 3505.5615234375, + -276.5766296386719 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 89 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect" + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 76, + "type": "VAELoaderMultiGPU", + "pos": [ + 2629.39599609375, + 275.80596923828125 + ], + "size": [ + 341.07464599609375, + 87.14057922363281 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 99 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoaderMultiGPU" + }, + "widgets_values": [ + "hunyuan_video_vae_bf16.safetensors", + "cuda:1" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 115, + "type": "EmptyHunyuanLatentVideo", + "pos": [ + 3026.87939453125, + 50.70602035522461 + ], + "size": [ + 210, + 130 + ], + "flags": { + "collapsed": false + }, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 155 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyHunyuanLatentVideo" + }, + "widgets_values": [ + 368, + 640, + 65, + 1 + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 2631.16015625, + -399.4529113769531 + ], + "size": [ + 353.56494140625, + 190.77996826171875 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "This workflow requires ComfyUI-GGUF", + "properties": {}, + "widgets_values": [ + "**⚠️ Dependency Alert! ⚠️**\n\nThis workflow relies on nodes from the [ComfyUI-GGUF](https://github.com/city96/ComfyUI-GGUF) custom node repository to function correctly. \n\nSpecifically:\n\n*\"CLIPLoaderGGUFMultiGPU\" \n\nwill not work without this dependency installed. Please install ComfyUI-GGUF before attempting to run this workflow." + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 68, + "type": "RandomNoise", + "pos": [ + 3244.127685546875, + -303.9937438964844 + ], + "size": [ + 250.37998962402344, + 82 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "NOISE", + "type": "NOISE", + "links": [ + 87 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "RandomNoise" + }, + "widgets_values": [ + 5770521, + "fixed" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 88, + "type": "UnetLoaderGGUFDisTorchMultiGPU", + "pos": [ + 2622.552734375, + -105.02156829833984 + ], + "size": [ + 358.005859375, + 128.5614013671875 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 141, + 142 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "UnetLoaderGGUFDisTorchMultiGPU" + }, + "widgets_values": [ + "hunyuan-video-t2v-720p-Q3_K_S.gguf", + "cuda:0", + "cuda:0,0.025;cuda:1,0.05;cpu,0.10" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 100, + "type": "DualCLIPLoaderGGUFDisTorchMultiGPU", + "pos": [ + 2623.7119140625, + 64.39619445800781 + ], + "size": [ + 348.5331726074219, + 154 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 154 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DualCLIPLoaderGGUFDisTorchMultiGPU" + }, + "widgets_values": [ + "clip_l.safetensors", + "llava-llama-3-8B-v1_1-Q4_K_M.gguf", + "hunyuan_video", + "cuda:1", + "cuda:0,0.025;cuda:1,0.05;cpu,0.10" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 70, + "type": "VHS_VideoCombine", + "pos": [ + 3809.87548828125, + -522.964111328125 + ], + "size": [ + 338.94024658203125, + 903.6536865234375 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 82 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 24, + "loop_count": 0, + "filename_prefix": "HunyuanVideo", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "HunyuanVideo_00314.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 24, + "workflow": "HunyuanVideo_00314.png", + "fullpath": "/home/johnj/ComfyUI/output/HunyuanVideo_00314.mp4" + }, + "muted": false + } + } + }, + { + "id": 64, + "type": "BasicGuider", + "pos": [ + 3288.76025390625, + -192.04156494140625 + ], + "size": [ + 222.3482666015625, + 46 + ], + "flags": { + "collapsed": true + }, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 78, + "slot_index": 0 + }, + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 79, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 88 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BasicGuider" + }, + "widgets_values": [] + }, + { + "id": 119, + "type": "Note", + "pos": [ + 2050.9873046875, + -145.64498901367188 + ], + "size": [ + 549.7151489257812, + 528.065185546875 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "How the DisTorch allocation string works", + "properties": {}, + "widgets_values": [ + "The allocation string uses a semicolon (;) to separate device allocations, and each device allocation has two parts separated by a comma:\n\nDevice name (e.g., \"cuda:0\", \"cuda:1\", \"cpu\")\nMemory fraction as a decimal (e.g., 0.025 = 2.5%, 0.05 = 5%, 0.10 = 10%)\n\n\nFor my defaults:\n\nCUDA GPU 0: Will allocate 2.5% of its total available memory\nCUDA GPU 1: Will allocate 5% of its total available memory\nCPU: Will allocate 10% of available system memory\n\n\nDisTorch then:\n\nCalculates how much actual memory this means for each device\nDistributes model layers across devices proportionally based on these allocations\nAssigns layers to minimize memory usage while maintaining model functionality\n\n\n\nSo if you have a model with 100 layers and your GPUs have the following memory:\n\nCUDA:0 has 24GB\nCUDA:1 has 24GB\nSystem has 64GB RAM\n\nThe allocation would be:\n\nCUDA:0: 600MB (2.5% of 24GB)\nCUDA:1: 1.2GB (5% of 24GB)\nCPU: 6.4GB (10% of 64GB)\n\nThis would help manage memory usage across your devices while keeping the model functional. The semicolon-separated format makes it easy to specify different allocations for different devices in a single string." + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 67, + "type": "CLIPTextEncode", + "pos": [ + 3032.827392578125, + -153.691650390625 + ], + "size": [ + 422.84503173828125, + 164.31304931640625 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 154 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 83 + ], + "slot_index": 0 + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "A light green frog with brown spots stares forward with its left eye moving subtly. The frog's skin is textured with small bumps, and its spots appear in irregular patterns across its head and face. The eye is round and black with a subtle shine. The head is slightly tilted to its left, and the body is barely visible. The background is a flat, solid bright yellow color, extending beyond the frog in all directions, with the lower part having a slight gradient into a lighter green. The camera zooms in slowly on the frog's eye and the frog looks directly at the viewer. The overall lighting is bright and diffused, creating a soft appearance." + ], + "color": "#232", + "bgcolor": "#353" + } + ], + "links": [ + [ + 78, + 73, + 0, + 64, + 0, + "MODEL" + ], + [ + 79, + 72, + 0, + 64, + 1, + "CONDITIONING" + ], + [ + 82, + 74, + 0, + 70, + 0, + "IMAGE" + ], + [ + 83, + 67, + 0, + 72, + 0, + "CONDITIONING" + ], + [ + 85, + 75, + 0, + 74, + 0, + "LATENT" + ], + [ + 87, + 68, + 0, + 75, + 0, + "NOISE" + ], + [ + 88, + 64, + 0, + 75, + 1, + "GUIDER" + ], + [ + 89, + 69, + 0, + 75, + 2, + "SAMPLER" + ], + [ + 90, + 66, + 0, + 75, + 3, + "SIGMAS" + ], + [ + 99, + 76, + 0, + 74, + 1, + "VAE" + ], + [ + 141, + 88, + 0, + 73, + 0, + "MODEL" + ], + [ + 142, + 88, + 0, + 66, + 0, + "MODEL" + ], + [ + 154, + 100, + 0, + 67, + 0, + "CLIP" + ], + [ + 155, + 115, + 0, + 75, + 4, + "LATENT" + ] + ], + "groups": [ + { + "id": 1, + "title": "GGUFDisTorchMultiGPU", + "bounding": [ + 2617.71240234375, + -173.0873565673828, + 360.4816589355469, + 537.903076171875 + ], + "color": "#8AA", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.797202450000122, + "offset": [ + -1830.6457542655762, + 790.7863966854694 + ] + }, + "ue_links": [], + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0 + }, + "version": 0.4 +} \ No newline at end of file diff --git a/examples/llamacpp_ltxv_2gpu_GGUF.json b/examples/llamacpp_ltxv_2gpu_gguf.json similarity index 100% rename from examples/llamacpp_ltxv_2gpu_GGUF.json rename to examples/llamacpp_ltxv_2gpu_gguf.json