nodes/chaojie-ComfyUI_StreamingT2V
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

README.md

Browse Source
This commit is contained in:
fles@qq.com
2024-04-10 10:03:16 +08:00
parent cf251847d2
commit ff2a857b5d
2 changed files with 911 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+11 -1
View File
@@ -8,6 +8,14 @@ VidXTend (<10G VRAM) | Original (<15G VRAM)
## workflow
[Recommended complete Workflow](https://github.com/chaojie/ComfyUI_StreamingT2V/blob/main/workflow/wf_final.json)
[Recommended STAGE 2 Workflow](https://github.com/chaojie/ComfyUI_StreamingT2V/blob/main/workflow/wf_VidXTend.json)
<img src="workflow/wf_VidXTend.png" raw=true>
<details> <summary>More Workflow</summary>
### Step 1 (It's not necessary.) Generate short video (16x256x256):
You can freely generate a 16x256x256 video without using the process below. You can even use videos downloaded from the internet.
@@ -60,4 +68,6 @@ https://github.com/chaojie/ComfyUI_StreamingT2V/blob/main/workflow/wf_svd.json
<img src="workflow/wf_svd.png" raw=true>
## [StreamingT2V](https://github.com/Picsart-AI-Research/StreamingT2V)
## [StreamingT2V](https://github.com/Picsart-AI-Research/StreamingT2V)
</details>
workflow/wf_final.json
+900
View File
@@ -0,0 +1,900 @@
{
"last_node_id": 20,
"last_link_id": 20,
"nodes": [
{
"id": 7,
"type": "CLIPTextEncode",
"pos": [
413,
389
],
"size": {
"0": 425.27801513671875,
"1": 180.6060791015625
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 5
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
6
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
""
]
},
{
"id": 5,
"type": "EmptyLatentImage",
"pos": [
473,
609
],
"size": {
"0": 315,
"1": 106
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
2
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "EmptyLatentImage"
},
"widgets_values": [
256,
256,
16
]
},
{
"id": 4,
"type": "CheckpointLoaderSimple",
"pos": [
71,
192
],
"size": {
"0": 315,
"1": 98
},
"flags": {},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
10
],
"slot_index": 0
},
{
"name": "CLIP",
"type": "CLIP",
"links": [
3,
5
],
"slot_index": 1
},
{
"name": "VAE",
"type": "VAE",
"links": [
8
],
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "CheckpointLoaderSimple"
},
"widgets_values": [
"Minimalism_v2.0.safetensors"
]
},
{
"id": 10,
"type": "ADE_AnimateDiffLoaderGen1",
"pos": [
70,
348
],
"size": {
"0": 315,
"1": 222
},
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 10
},
{
"name": "context_options",
"type": "CONTEXT_OPTIONS",
"link": null
},
{
"name": "motion_lora",
"type": "MOTION_LORA",
"link": null
},
{
"name": "ad_settings",
"type": "AD_SETTINGS",
"link": null
},
{
"name": "ad_keyframes",
"type": "AD_KEYFRAMES",
"link": null
},
{
"name": "sample_settings",
"type": "SAMPLE_SETTINGS",
"link": null
},
{
"name": "scale_multival",
"type": "MULTIVAL",
"link": null
},
{
"name": "effect_multival",
"type": "MULTIVAL",
"link": null
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
11
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ADE_AnimateDiffLoaderGen1"
},
"widgets_values": [
"animatediff_lightning_4step_comfyui.safetensors",
"sqrt_linear (AnimateDiff)"
]
},
{
"id": 8,
"type": "VAEDecode",
"pos": [
1209,
188
],
"size": {
"0": 210,
"1": 46
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7
},
{
"name": "vae",
"type": "VAE",
"link": 8
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
14
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
}
},
{
"id": 14,
"type": "StreamingT2VLoaderVidXTendModel",
"pos": [
1195,
51
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 2,
"mode": 0,
"outputs": [
{
"name": "VidXTendPipeline",
"type": "VidXTendPipeline",
"links": [
13
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "StreamingT2VLoaderVidXTendModel"
},
"widgets_values": [
"cuda"
]
},
{
"id": 16,
"type": "StreamingT2VLoaderEnhanceModel",
"pos": [
2388,
49
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 3,
"mode": 0,
"outputs": [
{
"name": "msxl_model",
"type": "msxl_model",
"links": [
15
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "StreamingT2VLoaderEnhanceModel"
},
"widgets_values": [
"cuda"
]
},
{
"id": 11,
"type": "VHS_VideoCombine",
"pos": [
2033,
199
],
"size": [
315,
599
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 16
},
{
"name": "audio",
"type": "VHS_AUDIO",
"link": null
},
{
"name": "batch_manager",
"type": "VHS_BatchManager",
"link": null
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": [
17
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 8,
"loop_count": 0,
"filename_prefix": "AnimateDiff",
"format": "video/h264-mp4",
"pix_fmt": "yuv420p",
"crf": 19,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "AnimateDiff_00038.mp4",
"subfolder": "",
"type": "output",
"format": "video/h264-mp4"
}
}
}
},
{
"id": 18,
"type": "VHS_FILENAMES_STRING_StreamingT2V",
"pos": [
2402,
194
],
"size": {
"0": 277.20001220703125,
"1": 26
},
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "filenames",
"type": "VHS_FILENAMES",
"link": 17
}
],
"outputs": [
{
"name": "STRING",
"type": "STRING",
"links": [
18
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VHS_FILENAMES_STRING_StreamingT2V"
},
"widgets_values": {}
},
{
"id": 6,
"type": "CLIPTextEncode",
"pos": [
415,
186
],
"size": {
"0": 422.84503173828125,
"1": 164.31304931640625
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 3
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
4
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"A girl smiling"
]
},
{
"id": 20,
"type": "VHS_LoadVideoPath",
"pos": [
3151.8190696022725,
176.27268635142931
],
"size": [
317.4000244140625,
214
],
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "batch_manager",
"type": "VHS_BatchManager",
"link": null
},
{
"name": "video",
"type": "STRING",
"link": 19,
"widget": {
"name": "video"
}
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
20
],
"shape": 3,
"slot_index": 0
},
{
"name": "frame_count",
"type": "INT",
"links": null,
"shape": 3
},
{
"name": "audio",
"type": "VHS_AUDIO",
"links": null,
"shape": 3
},
{
"name": "video_info",
"type": "VHS_VIDEOINFO",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "VHS_LoadVideoPath"
},
"widgets_values": {
"video": "X://insert/path/here.mp4",
"force_rate": 0,
"force_size": "Disabled",
"custom_width": 512,
"custom_height": 512,
"frame_load_cap": 0,
"skip_first_frames": 0,
"select_every_nth": 1,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"frame_load_cap": 0,
"skip_first_frames": 0,
"force_rate": 0,
"filename": "X://insert/path/here.mp4",
"type": "path",
"format": "video/mp4",
"select_every_nth": 1
}
}
}
},
{
"id": 3,
"type": "KSampler",
"pos": [
863,
186
],
"size": {
"0": 315,
"1": 262
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 11
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 4
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 6
},
{
"name": "latent_image",
"type": "LATENT",
"link": 2
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
7
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "KSampler"
},
"widgets_values": [
309440923853978,
"fixed",
4,
1,
"euler",
"sgm_uniform",
1
]
},
{
"id": 17,
"type": "StreamingT2VRunEnhanceStep",
"pos": [
2738,
216
],
"size": {
"0": 315,
"1": 222
},
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "msxl_model",
"type": "msxl_model",
"link": 15
},
{
"name": "low_video_path",
"type": "STRING",
"link": 18,
"widget": {
"name": "low_video_path"
}
}
],
"outputs": [
{
"name": "video_path",
"type": "STRING",
"links": [
19
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "StreamingT2VRunEnhanceStep"
},
"widgets_values": [
"",
"A girl smiling",
24,
56,
32,
1280,
720,
true
]
},
{
"id": 19,
"type": "VHS_VideoCombine",
"pos": [
3557,
78
],
"size": [
315,
599
],
"flags": {},
"order": 14,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 20
},
{
"name": "audio",
"type": "VHS_AUDIO",
"link": null
},
{
"name": "batch_manager",
"type": "VHS_BatchManager",
"link": null
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 25,
"loop_count": 0,
"filename_prefix": "AnimateDiff",
"format": "video/h264-mp4",
"pix_fmt": "yuv420p",
"crf": 19,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "AnimateDiff_00039.mp4",
"subfolder": "",
"type": "output",
"format": "video/h264-mp4"
}
}
}
},
{
"id": 15,
"type": "StreamingT2VRunLongStepVidXTendPipeline",
"pos": [
1658,
204
],
"size": {
"0": 327.6000061035156,
"1": 222
},
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "VidXTendPipeline",
"type": "VidXTendPipeline",
"link": 13
},
{
"name": "short_video",
"type": "IMAGE",
"link": 14
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
16
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "StreamingT2VRunLongStepVidXTendPipeline"
},
"widgets_values": [
"A girl smiling",
24,
50,
9,
1774,
"fixed",
"worst quality, normal quality, low quality, low res, blurry, text,watermark, logo, banner, extra digits, cropped,jpeg artifacts, signature, username, error,sketch ,duplicate, ugly, monochrome, horror, geometry, mutation, disgusting"
]
}
],
"links": [
[
2,
5,
0,
3,
3,
"LATENT"
],
[
3,
4,
1,
6,
0,
"CLIP"
],
[
4,
6,
0,
3,
1,
"CONDITIONING"
],
[
5,
4,
1,
7,
0,
"CLIP"
],
[
6,
7,
0,
3,
2,
"CONDITIONING"
],
[
7,
3,
0,
8,
0,
"LATENT"
],
[
8,
4,
2,
8,
1,
"VAE"
],
[
10,
4,
0,
10,
0,
"MODEL"
],
[
11,
10,
0,
3,
0,
"MODEL"
],
[
13,
14,
0,
15,
0,
"VidXTendPipeline"
],
[
14,
8,
0,
15,
1,
"IMAGE"
],
[
15,
16,
0,
17,
0,
"msxl_model"
],
[
16,
15,
0,
11,
0,
"IMAGE"
],
[
17,
11,
0,
18,
0,
"VHS_FILENAMES"
],
[
18,
18,
0,
17,
1,
"STRING"
],
[
19,
17,
0,
20,
1,
"STRING"
],
[
20,
20,
0,
19,
0,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {},
"version": 0.4
}