From aa6e296ea8b8cc69528b93e0813da4ef76e68177 Mon Sep 17 00:00:00 2001 From: flowtone Date: Sat, 16 Mar 2024 01:26:16 +0200 Subject: [PATCH] add low vram workflows, update readme with troubleshooting steps --- README.md | 18 ++- low-vram/1.json | 423 ++++++++++++++++++++++++++++++++++++++++++++++++ low-vram/2.json | 207 ++++++++++++++++++++++++ low-vram/3.json | 281 ++++++++++++++++++++++++++++++++ low-vram/4.json | 295 +++++++++++++++++++++++++++++++++ 5 files changed, 1221 insertions(+), 3 deletions(-) create mode 100644 low-vram/1.json create mode 100644 low-vram/2.json create mode 100644 low-vram/3.json create mode 100644 low-vram/4.json diff --git a/README.md b/README.md index 3640910..bd96e46 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,7 @@ This node has been adapted from the [official implementation](https://github.com * Use the example workflow ([cpu](workflow_rembg_crm.json) | [cuda](workflow_rembg_crm_cuda.json)) * Note: To run the example workflows you will also need to install [ComfyUI_essentials](https://github.com/cubiq/ComfyUI_essentials) - -Acknowledgement: +### Acknowledgement: * Researchers of CRM: [Zhengyi Wang](https://thuwzy.github.io/), Yikai Wang, Yifei Chen, Chendong Xiang, Shuo Chen, Dajiang Yu, Chongxuan Li, Hang Su, Jun Zhu * three.js comfyui interface: [MrForExample](https://github.com/MrForExample/ComfyUI-3D-Pack) @@ -49,4 +48,17 @@ This is a community project from [flowt.ai](https://flowt.ai). If you like it, c flowt.ai logo - \ No newline at end of file + + +### Troubleshooting +* NVdiffrast + * Windows users with cuda(nvidia) gpus might need some special preparations to make the `nvdiffrast` module work. Please check the official documentation of nvdiffrast for [windows installation instructions](https://nvlabs.github.io/nvdiffrast/#windows). + * `CUDA_HOME environment variable is not set` - you will need to set an environment variable that points to your cuda installation path. [See this stackoverflow answer](https://stackoverflow.com/a/58890075). + * If you're experiencing too many issues trying to install NVdiffrast, consider using the cpu workflow by restarting comfyui with the `cpu-only` option (much slower). +* `Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!` + * This usually happens if you tried to run the cpu workflow but have a cuda gpu. Try to restart comfyui and run only the cuda workflow. +* Portable ComfyUI + * Users might need to install the dependencies differently, [see here](https://github.com/flowtyone/ComfyUI-Flowty-CRM/issues/3#issuecomment-2000450286). +* Not enough VRAM/RAM + * Using these nodes you should be able to run CRM on GPUs with 8GB of VRAM and above, and at least 16GB of RAM. + * For GPUs with less than 16GB of VRAM, you might need to split the workflow into 4 parts and run them separately. [I've added workflows here for demonstration.](low-vram) \ No newline at end of file diff --git a/low-vram/1.json b/low-vram/1.json new file mode 100644 index 0000000..93e836f --- /dev/null +++ b/low-vram/1.json @@ -0,0 +1,423 @@ +{ + "last_node_id": 51, + "last_link_id": 77, + "nodes": [ + { + "id": 31, + "type": "ImageScale", + "pos": [ + 503, + 941 + ], + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 38 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 39 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageScale" + }, + "widgets_values": [ + "nearest-exact", + 1024, + 1024, + "center" + ] + }, + { + "id": 21, + "type": "RemBGSession+", + "pos": [ + 504, + 800 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "REMBG_SESSION", + "type": "REMBG_SESSION", + "links": [ + 22 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "RemBGSession+" + }, + "widgets_values": [ + "u2net: general purpose", + "CPU" + ] + }, + { + "id": 20, + "type": "ImageRemoveBackground+", + "pos": [ + 841, + 801 + ], + "size": { + "0": 241.79998779296875, + "1": 46 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "rembg_session", + "type": "REMBG_SESSION", + "link": 22, + "slot_index": 0 + }, + { + "name": "image", + "type": "IMAGE", + "link": 39 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 33, + 45 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 46, + 48 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "ImageRemoveBackground+" + } + }, + { + "id": 32, + "type": "PreviewImage", + "pos": [ + 1110, + 929 + ], + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 47 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 23, + "type": "PreviewImage", + "pos": [ + 857, + 930 + ], + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 45 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 26, + "type": "MaskPreview+", + "pos": [ + 857, + 1221 + ], + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 46 + } + ], + "properties": { + "Node name for S&R": "MaskPreview+" + } + }, + { + "id": 17, + "type": "LoadImage", + "pos": [ + 167, + 941 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": { + "collapsed": false + }, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 38 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "turtle.jpeg", + "image" + ] + }, + { + "id": 28, + "type": "CRMPreprocessForPoser", + "pos": [ + 1107, + 804 + ], + "size": { + "0": 317.4000244140625, + "1": 78 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "reference_image", + "type": "IMAGE", + "link": 33, + "slot_index": 0 + }, + { + "name": "reference_mask", + "type": "MASK", + "link": 48, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "processed", + "type": "IMAGE", + "links": [ + 47, + 77 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CRMPreprocessForPoser" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 50, + "type": "SaveImage", + "pos": [ + 1469, + 809 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 77 + } + ], + "properties": {}, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 51, + "type": "Note", + "pos": [ + 513, + 1134 + ], + "size": [ + 236.62061950683608, + 92.81676513671891 + ], + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Here you should feed your input image for preprocessing. Then find the saved image in comfyui's output folder for processing in the next workflows" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 22, + 21, + 0, + 20, + 0, + "REMBG_SESSION" + ], + [ + 33, + 20, + 0, + 28, + 0, + "IMAGE" + ], + [ + 38, + 17, + 0, + 31, + 0, + "IMAGE" + ], + [ + 39, + 31, + 0, + 20, + 1, + "IMAGE" + ], + [ + 45, + 20, + 0, + 23, + 0, + "IMAGE" + ], + [ + 46, + 20, + 1, + 26, + 0, + "MASK" + ], + [ + 47, + 28, + 0, + 32, + 0, + "IMAGE" + ], + [ + 48, + 20, + 1, + 28, + 1, + "MASK" + ], + [ + 77, + 28, + 0, + 50, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/low-vram/2.json b/low-vram/2.json new file mode 100644 index 0000000..879d564 --- /dev/null +++ b/low-vram/2.json @@ -0,0 +1,207 @@ +{ + "last_node_id": 50, + "last_link_id": 75, + "nodes": [ + { + "id": 49, + "type": "LoadImage", + "pos": [ + 567, + 873 + ], + "size": [ + 315, + 314.00000381469727 + ], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 74 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ComfyUI_00012_.png", + "image" + ] + }, + { + "id": 44, + "type": "CRMPoserConfig", + "pos": [ + 914, + 876 + ], + "size": { + "0": 393, + "1": 130 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "processed_image", + "type": "IMAGE", + "link": 74, + "slot_index": 0 + } + ], + "outputs": [ + { + "name": "CRM_POSE_CONFIG", + "type": "CRM_POSE_CONFIG", + "links": [ + 62 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CRMPoserConfig" + }, + "widgets_values": [ + 1234, + "fixed", + 5.5, + 30 + ] + }, + { + "id": 46, + "type": "CRMPoseSampler", + "pos": [ + 1338, + 880 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "config", + "type": "CRM_POSE_CONFIG", + "link": 62 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 75 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CRMPoseSampler" + }, + "widgets_values": [ + "pixel-diffusion.pth" + ] + }, + { + "id": 34, + "type": "SaveImage", + "pos": [ + 1696, + 882 + ], + "size": { + "0": 315, + "1": 270 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 75, + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 50, + "type": "Note", + "pos": [ + 916, + 1054 + ], + "size": [ + 258.10373352050783, + 95.24444946289077 + ], + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Here load the preprocessed image from workflow 1. After it finishes, find the 6 output images and copy them into a new folder called 'poses' inside the comfyui's input folder." + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 62, + 44, + 0, + 46, + 0, + "CRM_POSE_CONFIG" + ], + [ + 74, + 49, + 0, + 44, + 0, + "IMAGE" + ], + [ + 75, + 46, + 0, + 34, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/low-vram/3.json b/low-vram/3.json new file mode 100644 index 0000000..1d63f17 --- /dev/null +++ b/low-vram/3.json @@ -0,0 +1,281 @@ +{ + "last_node_id": 52, + "last_link_id": 78, + "nodes": [ + { + "id": 49, + "type": "LoadImage", + "pos": [ + 567, + 873 + ], + "size": [ + 315, + 314.00000381469727 + ], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 74 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ComfyUI_00012_.png", + "image" + ] + }, + { + "id": 34, + "type": "SaveImage", + "pos": [ + 1696, + 882 + ], + "size": { + "0": 315, + "1": 270 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 77, + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 44, + "type": "CRMPoserConfig", + "pos": [ + 914, + 876 + ], + "size": { + "0": 393, + "1": 130 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "processed_image", + "type": "IMAGE", + "link": 74, + "slot_index": 0 + } + ], + "outputs": [ + { + "name": "CRM_POSE_CONFIG", + "type": "CRM_POSE_CONFIG", + "links": [ + 76 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CRMPoserConfig" + }, + "widgets_values": [ + 1234, + "fixed", + 5.5, + 30 + ] + }, + { + "id": 51, + "type": "CCMSampler", + "pos": [ + 1336, + 877 + ], + "size": { + "0": 315, + "1": 78 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "config", + "type": "CRM_POSE_CONFIG", + "link": 76 + }, + { + "name": "poses", + "type": "IMAGE", + "link": 78, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 77 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CCMSampler" + }, + "widgets_values": [ + "ccm-diffusion.pth" + ] + }, + { + "id": 52, + "type": "VHS_LoadImagesPath", + "pos": [ + 916, + 1050 + ], + "size": [ + 226.8000030517578, + 194 + ], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 78 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "INT", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadImagesPath" + }, + "widgets_values": { + "directory": "input/poses/", + "image_load_cap": 0, + "skip_first_images": 0, + "select_every_nth": 1, + "choose folder to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_images": 0, + "filename": "input/poses/", + "type": "path", + "format": "folder", + "select_every_nth": 1 + } + } + } + }, + { + "id": 50, + "type": "Note", + "pos": [ + 597, + 1232 + ], + "size": [ + 279.3755228342102, + 122.07330661604556 + ], + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Here load the preprocessed image from workflow 1. In the 'load images' node, select the path of the 'poses' images saved from workflow 2. after it finishes, find the 6 output images and copy them into a new folder called 'ccm' inside the comfyui's input folder." + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 74, + 49, + 0, + 44, + 0, + "IMAGE" + ], + [ + 76, + 44, + 0, + 51, + 0, + "CRM_POSE_CONFIG" + ], + [ + 77, + 51, + 0, + 34, + 0, + "IMAGE" + ], + [ + 78, + 52, + 0, + 51, + 1, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/low-vram/4.json b/low-vram/4.json new file mode 100644 index 0000000..3b86c2b --- /dev/null +++ b/low-vram/4.json @@ -0,0 +1,295 @@ +{ + "last_node_id": 51, + "last_link_id": 75, + "nodes": [ + { + "id": 37, + "type": "CRMModelLoader", + "pos": [ + 1525, + 692 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "CRM_MODEL", + "type": "CRM_MODEL", + "links": [ + 68 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CRMModelLoader" + }, + "widgets_values": [ + "CRM.pth" + ] + }, + { + "id": 48, + "type": "CRMViewer", + "pos": [ + 1863, + 929 + ], + "size": [ + 600, + 500 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "mesh", + "type": "MESH", + "link": 72 + } + ], + "properties": { + "Node name for S&R": "CRMViewer" + }, + "widgets_values": [ + true + ] + }, + { + "id": 47, + "type": "CRMModelerCuda", + "pos": [ + 1861, + 810 + ], + "size": { + "0": 210, + "1": 66 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "crm_model", + "type": "CRM_MODEL", + "link": 68, + "slot_index": 0 + }, + { + "name": "poses", + "type": "IMAGE", + "link": 74, + "slot_index": 1 + }, + { + "name": "coordinates", + "type": "IMAGE", + "link": 75, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "MESH", + "type": "MESH", + "links": [ + 72 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CRMModelerCuda" + } + }, + { + "id": 49, + "type": "VHS_LoadImagesPath", + "pos": [ + 1498, + 829 + ], + "size": [ + 226.8000030517578, + 194 + ], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 74 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "INT", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadImagesPath" + }, + "widgets_values": { + "directory": "/input/poses", + "image_load_cap": 0, + "skip_first_images": 0, + "select_every_nth": 1, + "choose folder to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_images": 0, + "filename": "/input/poses", + "type": "path", + "format": "folder", + "select_every_nth": 1 + } + } + } + }, + { + "id": 50, + "type": "VHS_LoadImagesPath", + "pos": [ + 1500.5979614257812, + 1077.2034412730823 + ], + "size": [ + 226.8000030517578, + 194 + ], + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 75 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "INT", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadImagesPath" + }, + "widgets_values": { + "directory": "/input/ccm", + "image_load_cap": 0, + "skip_first_images": 0, + "select_every_nth": 1, + "choose folder to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_images": 0, + "filename": "/input/ccm", + "type": "path", + "format": "folder", + "select_every_nth": 1 + } + } + } + }, + { + "id": 51, + "type": "Note", + "pos": [ + 1505, + 1323 + ], + "size": [ + 233.144198330966, + 109.81053577769876 + ], + "flags": {}, + "order": 3, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Here, load the images from workflow 2 and 3 in their respective places. Then you should be able to generate the final model and preview it here." + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 68, + 37, + 0, + 47, + 0, + "CRM_MODEL" + ], + [ + 72, + 47, + 0, + 48, + 0, + "MESH" + ], + [ + 74, + 49, + 0, + 47, + 1, + "IMAGE" + ], + [ + 75, + 50, + 0, + 47, + 2, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file