From ff2a857b5da430b7a6773dc4196502db36154aff Mon Sep 17 00:00:00 2001 From: "fles@qq.com" Date: Wed, 10 Apr 2024 10:03:16 +0800 Subject: [PATCH] README.md --- README.md | 12 +- workflow/wf_final.json | 900 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 911 insertions(+), 1 deletion(-) create mode 100644 workflow/wf_final.json diff --git a/README.md b/README.md index acfca53..f348c03 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,14 @@ VidXTend (<10G VRAM) | Original (<15G VRAM) ## workflow +[Recommended complete Workflow](https://github.com/chaojie/ComfyUI_StreamingT2V/blob/main/workflow/wf_final.json) + +[Recommended STAGE 2 Workflow](https://github.com/chaojie/ComfyUI_StreamingT2V/blob/main/workflow/wf_VidXTend.json) + + + +
More Workflow + ### Step 1 (It's not necessary.) Generate short video (16x256x256): You can freely generate a 16x256x256 video without using the process below. You can even use videos downloaded from the internet. @@ -60,4 +68,6 @@ https://github.com/chaojie/ComfyUI_StreamingT2V/blob/main/workflow/wf_svd.json -## [StreamingT2V](https://github.com/Picsart-AI-Research/StreamingT2V) \ No newline at end of file +## [StreamingT2V](https://github.com/Picsart-AI-Research/StreamingT2V) + +
\ No newline at end of file diff --git a/workflow/wf_final.json b/workflow/wf_final.json new file mode 100644 index 0000000..206de29 --- /dev/null +++ b/workflow/wf_final.json @@ -0,0 +1,900 @@ +{ + "last_node_id": 20, + "last_link_id": 20, + "nodes": [ + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 413, + 389 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 473, + 609 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 256, + 256, + 16 + ] + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 71, + 192 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 10 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "Minimalism_v2.0.safetensors" + ] + }, + { + "id": 10, + "type": "ADE_AnimateDiffLoaderGen1", + "pos": [ + 70, + 348 + ], + "size": { + "0": 315, + "1": 222 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 10 + }, + { + "name": "context_options", + "type": "CONTEXT_OPTIONS", + "link": null + }, + { + "name": "motion_lora", + "type": "MOTION_LORA", + "link": null + }, + { + "name": "ad_settings", + "type": "AD_SETTINGS", + "link": null + }, + { + "name": "ad_keyframes", + "type": "AD_KEYFRAMES", + "link": null + }, + { + "name": "sample_settings", + "type": "SAMPLE_SETTINGS", + "link": null + }, + { + "name": "scale_multival", + "type": "MULTIVAL", + "link": null + }, + { + "name": "effect_multival", + "type": "MULTIVAL", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 11 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffLoaderGen1" + }, + "widgets_values": [ + "animatediff_lightning_4step_comfyui.safetensors", + "sqrt_linear (AnimateDiff)" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1209, + 188 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 14 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 14, + "type": "StreamingT2VLoaderVidXTendModel", + "pos": [ + 1195, + 51 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "VidXTendPipeline", + "type": "VidXTendPipeline", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "StreamingT2VLoaderVidXTendModel" + }, + "widgets_values": [ + "cuda" + ] + }, + { + "id": 16, + "type": "StreamingT2VLoaderEnhanceModel", + "pos": [ + 2388, + 49 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "msxl_model", + "type": "msxl_model", + "links": [ + 15 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "StreamingT2VLoaderEnhanceModel" + }, + "widgets_values": [ + "cuda" + ] + }, + { + "id": 11, + "type": "VHS_VideoCombine", + "pos": [ + 2033, + 199 + ], + "size": [ + 315, + 599 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 16 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null + }, + { + "name": "batch_manager", + "type": "VHS_BatchManager", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": [ + 17 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00038.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4" + } + } + } + }, + { + "id": 18, + "type": "VHS_FILENAMES_STRING_StreamingT2V", + "pos": [ + 2402, + 194 + ], + "size": { + "0": 277.20001220703125, + "1": 26 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "filenames", + "type": "VHS_FILENAMES", + "link": 17 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 18 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_FILENAMES_STRING_StreamingT2V" + }, + "widgets_values": {} + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 415, + 186 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "A girl smiling" + ] + }, + { + "id": 20, + "type": "VHS_LoadVideoPath", + "pos": [ + 3151.8190696022725, + 176.27268635142931 + ], + "size": [ + 317.4000244140625, + 214 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "batch_manager", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "video", + "type": "STRING", + "link": 19, + "widget": { + "name": "video" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 20 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoPath" + }, + "widgets_values": { + "video": "X://insert/path/here.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "X://insert/path/here.mp4", + "type": "path", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 863, + 186 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 11 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 309440923853978, + "fixed", + 4, + 1, + "euler", + "sgm_uniform", + 1 + ] + }, + { + "id": 17, + "type": "StreamingT2VRunEnhanceStep", + "pos": [ + 2738, + 216 + ], + "size": { + "0": 315, + "1": 222 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "msxl_model", + "type": "msxl_model", + "link": 15 + }, + { + "name": "low_video_path", + "type": "STRING", + "link": 18, + "widget": { + "name": "low_video_path" + } + } + ], + "outputs": [ + { + "name": "video_path", + "type": "STRING", + "links": [ + 19 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "StreamingT2VRunEnhanceStep" + }, + "widgets_values": [ + "", + "A girl smiling", + 24, + 56, + 32, + 1280, + 720, + true + ] + }, + { + "id": 19, + "type": "VHS_VideoCombine", + "pos": [ + 3557, + 78 + ], + "size": [ + 315, + 599 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 20 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null + }, + { + "name": "batch_manager", + "type": "VHS_BatchManager", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 25, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00039.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4" + } + } + } + }, + { + "id": 15, + "type": "StreamingT2VRunLongStepVidXTendPipeline", + "pos": [ + 1658, + 204 + ], + "size": { + "0": 327.6000061035156, + "1": 222 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "VidXTendPipeline", + "type": "VidXTendPipeline", + "link": 13 + }, + { + "name": "short_video", + "type": "IMAGE", + "link": 14 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "StreamingT2VRunLongStepVidXTendPipeline" + }, + "widgets_values": [ + "A girl smiling", + 24, + 50, + 9, + 1774, + "fixed", + "worst quality, normal quality, low quality, low res, blurry, text,watermark, logo, banner, extra digits, cropped,jpeg artifacts, signature, username, error,sketch ,duplicate, ugly, monochrome, horror, geometry, mutation, disgusting" + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 10, + 4, + 0, + 10, + 0, + "MODEL" + ], + [ + 11, + 10, + 0, + 3, + 0, + "MODEL" + ], + [ + 13, + 14, + 0, + 15, + 0, + "VidXTendPipeline" + ], + [ + 14, + 8, + 0, + 15, + 1, + "IMAGE" + ], + [ + 15, + 16, + 0, + 17, + 0, + "msxl_model" + ], + [ + 16, + 15, + 0, + 11, + 0, + "IMAGE" + ], + [ + 17, + 11, + 0, + 18, + 0, + "VHS_FILENAMES" + ], + [ + 18, + 18, + 0, + 17, + 1, + "STRING" + ], + [ + 19, + 17, + 0, + 20, + 1, + "STRING" + ], + [ + 20, + 20, + 0, + 19, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file