diff --git a/README.md b/README.md index 95994f5..dece9f3 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ ## 🔆 New Features/Updates +- ✅ `11/12/2024` Added a newly fine-tuned version of [`Animatediff`](https://huggingface.co/songkey/hm_animatediff_frame12) with a patch size of 12, which uses less VRAM (Tested on 2080Ti). - ✅ `11/11/2024` Optimized VRAM usage and added `HMVideoSimplePipeline` (`workflows/hellomeme_video_simple_workflow.json`), which doesn’t use Animatediff and can run on machines with less than 12G VRAM. - ✅ `11/6/2024` The face proportion in the reference image significantly affects the generation quality. We have encapsulated the **recommended image cropping method** used during training into a `CropReferenceImage` Node. Refer to the workflows in the `ComfyUI_HelloMeme/workflows directory`: `hellomeme_video_cropref_workflow.json` and `hellomeme_image_cropref_workflow.json`. diff --git a/hellomeme/pipelines/pipline_hm_image.py b/hellomeme/pipelines/pipline_hm_image.py index 1b16611..3d11379 100644 --- a/hellomeme/pipelines/pipline_hm_image.py +++ b/hellomeme/pipelines/pipline_hm_image.py @@ -26,7 +26,7 @@ from ..models import HMDenoising3D, HMControlNet from ..models import HMReferenceAdapter class HMImagePipeline(StableDiffusionImg2ImgPipeline): - def caryomitosis(self,): + def caryomitosis(self, **kwargs): if not hasattr(self, "unet_ref"): self.unet_ref = HMDenoising3D.from_unet2d(self.unet) if not isinstance(self.unet, HMDenoising3D): diff --git a/hellomeme/pipelines/pipline_hm_video.py b/hellomeme/pipelines/pipline_hm_video.py index 4418251..0715e67 100644 --- a/hellomeme/pipelines/pipline_hm_video.py +++ b/hellomeme/pipelines/pipline_hm_video.py @@ -27,13 +27,17 @@ from ..utils import dicts_to_device, cat_dicts class HMVideoPipeline(StableDiffusionImg2ImgPipeline): - def caryomitosis(self,): + def caryomitosis(self, patch_frames=12, **kwargs): if hasattr(self, "unet_ref"): del self.unet_ref self.unet_ref = HMDenoising3D.from_unet2d(self.unet) - self.num_frames = 16 - adapter = MotionAdapter.from_pretrained("songkey/hm_animatediff", torch_dtype=torch.float16) + if patch_frames >= 14: + self.num_frames = 16 + adapter = MotionAdapter.from_pretrained("songkey/hm_animatediff", torch_dtype=torch.float16) + else: + self.num_frames = 12 + adapter = MotionAdapter.from_pretrained("songkey/hm_animatediff_frame12", torch_dtype=torch.float16) unet = HMDenoisingMotion.from_unet2d(unet=self.unet, motion_adapter=adapter, load_weights=True) # todo: 不够优雅 del self.unet diff --git a/hellomeme/pipelines/pipline_hm_video_simple.py b/hellomeme/pipelines/pipline_hm_video_simple.py index ebd313a..d9c570a 100644 --- a/hellomeme/pipelines/pipline_hm_video_simple.py +++ b/hellomeme/pipelines/pipline_hm_video_simple.py @@ -26,7 +26,7 @@ from ..models import HMReferenceAdapter from ..utils import dicts_to_device, cat_dicts class HMVideoSimplePipeline(StableDiffusionImg2ImgPipeline): - def caryomitosis(self,): + def caryomitosis(self, **kwargs): if hasattr(self, "unet_ref"): del self.unet_ref self.unet_ref = HMDenoising3D.from_unet2d(self.unet) diff --git a/hellomeme/utils.py b/hellomeme/utils.py index 82bd707..02d783d 100644 --- a/hellomeme/utils.py +++ b/hellomeme/utils.py @@ -413,7 +413,7 @@ def load_data_list(data_dir, post_fix='.pickle;.txt'): ret_list = [] for root, dirnames, filenames in os.walk(data_dir): for name in filenames: - if os.path.splitext(name)[1] in post_fixs and not name.startswith('.'): + if os.path.splitext(name)[1].lower() in post_fixs and not name.startswith('.'): data_path = os.path.join(root, name) ret_list.append(data_path) return ret_list \ No newline at end of file diff --git a/meme.py b/meme.py index 4570936..fd9593c 100644 --- a/meme.py +++ b/meme.py @@ -49,7 +49,7 @@ class HMImagePipelineLoader: return { "optional": { - "checkpoint_path": (['None'] + checkpoint_files, ), + "checkpoint_path": (['SD1.5'] + checkpoint_files, ), "lora_path": (['None'] + lora_files, ), "gpu_id": ("INT", {"default": 0}), } @@ -96,9 +96,10 @@ class HMVideoPipelineLoader: return { "optional": { - "checkpoint_path": (['None'] + checkpoint_files, ), + "checkpoint_path": (['SD1.5'] + checkpoint_files, ), "lora_path": (['None'] + lora_files, ), - "gpu_id": ("INT", {"default": 0}), + "patch_frames": ([12, 16], ), + "gpu_id": ("INT", {"default": 0}, ), } } @@ -107,7 +108,7 @@ class HMVideoPipelineLoader: FUNCTION = "load_pipeline" CATEGORY = "hellomeme" - def load_pipeline(self, checkpoint_path=None, lora_path=None, gpu_id=0): + def load_pipeline(self, checkpoint_path=None, lora_path=None, patch_frames=12, gpu_id=0): dtype = torch.float16 if gpu_id >= 0: device = torch.device("cuda:{}".format(gpu_id)) @@ -115,7 +116,7 @@ class HMVideoPipelineLoader: device = torch.device("cpu") pipeline = HMVideoPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5").to(dtype=dtype, device=device) - pipeline.caryomitosis() + pipeline.caryomitosis(patch_frames=patch_frames) ### load customized checkpoint or lora here: ## checkpoints @@ -146,7 +147,7 @@ class HMVideoSimplePipelineLoader: return { "optional": { - "checkpoint_path": (['None'] + checkpoint_files, ), + "checkpoint_path": (['SD1.5'] + checkpoint_files, ), "lora_path": (['None'] + lora_files, ), "gpu_id": ("INT", {"default": 0}), } @@ -217,8 +218,8 @@ class GetReferenceImageRT: def INPUT_TYPES(s): return { "required": { - "face_toolkits": ("FACE_TOOLKITS",), "image": ("IMAGE",), + "face_toolkits": ("FACE_TOOLKITS",), } } @@ -227,7 +228,7 @@ class GetReferenceImageRT: FUNCTION = "get_reference_image_rt" CATEGORY = "hellomeme" - def get_reference_image_rt(self, face_toolkits, image): + def get_reference_image_rt(self, image, face_toolkits): image_np = cv2.cvtColor((image[0] * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) image_np = cv2.resize(image_np, (512, 512)) # print(image_np.shape) diff --git a/workflows/hellomeme_image_workflow.json b/workflows/hellomeme_image_workflow.json index 7816df7..e5bd7ab 100644 --- a/workflows/hellomeme_image_workflow.json +++ b/workflows/hellomeme_image_workflow.json @@ -1,7 +1,34 @@ { - "last_node_id": 37, - "last_link_id": 55, + "last_node_id": 38, + "last_link_id": 58, "nodes": [ + { + "id": 26, + "type": "PreviewImage", + "pos": { + "0": 795, + "1": 121 + }, + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 34 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, { "id": 8, "type": "LoadImage", @@ -41,147 +68,76 @@ ] }, { - "id": 2, - "type": "HMFaceToolkitsLoader", + "id": 36, + "type": "LoadImage", "pos": { - "0": 61, - "1": 697 - }, - "size": { - "0": 315, - "1": 58 + "0": 65, + "1": 871 }, + "size": [ + 315, + 314 + ], "flags": {}, "order": 1, "mode": 0, "inputs": [], "outputs": [ { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", + "name": "IMAGE", + "type": "IMAGE", "links": [ - 13, - 32, - 42 + 53 ], "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null } ], "properties": { - "Node name for S&R": "HMFaceToolkitsLoader" + "Node name for S&R": "LoadImage" }, "widgets_values": [ - 0 + "yao.jpg", + "image" ] }, { - "id": 26, - "type": "PreviewImage", + "id": 35, + "type": "HMImagePipelineLoader", "pos": { - "0": 795, - "1": 121 + "0": 65, + "1": 106 }, "size": { - "0": 210, - "1": 246 + "0": 352.79998779296875, + "1": 106 }, "flags": {}, - "order": 6, + "order": 2, "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 34 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, - { - "id": 11, - "type": "GetReferenceImageRT", - "pos": { - "0": 749, - "1": 546 - }, - "size": { - "0": 241.79998779296875, - "1": 46 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 13 - }, - { - "name": "image", - "type": "IMAGE", - "link": 37 - } - ], + "inputs": [], "outputs": [ { - "name": "ref_rt", - "type": "REFRT", + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", "links": [ - 50 + 54 ], "slot_index": 0 } ], "properties": { - "Node name for S&R": "GetReferenceImageRT" + "Node name for S&R": "HMImagePipelineLoader" }, - "widgets_values": [] - }, - { - "id": 31, - "type": "GetVideoDriveParams", - "pos": { - "0": 447, - "1": 745 - }, - "size": { - "0": 330, - "1": 46 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 42 - }, - { - "name": "images", - "type": "IMAGE", - "link": 54 - } - ], - "outputs": [ - { - "name": "drive_params", - "type": "DRIVE_PARAMS", - "links": [ - 51 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetVideoDriveParams" - }, - "widgets_values": [] + "widgets_values": [ + "SD1.5", + "None", + 0 + ] }, { "id": 25, @@ -215,8 +171,8 @@ "type": "IMAGE", "links": [ 34, - 37, - 52 + 50, + 55 ], "slot_index": 0 } @@ -227,70 +183,127 @@ "widgets_values": [] }, { - "id": 34, - "type": "HMPipelineImage", + "id": 2, + "type": "HMFaceToolkitsLoader", "pos": { - "0": 1119.219970703125, - "1": 376.4541015625 + "0": 61, + "1": 697 }, "size": { "0": 315, - "1": 262 + "1": 58 }, "flags": {}, - "order": 8, + "order": 3, "mode": 0, - "inputs": [ - { - "name": "hm_image_pipeline", - "type": "HMIMAGEPIPELINE", - "link": 53 - }, - { - "name": "ref_image", - "type": "IMAGE", - "link": 52 - }, - { - "name": "ref_rt", - "type": "REFRT", - "link": 50 - }, - { - "name": "drive_params", - "type": "DRIVE_PARAMS", - "link": 51 - } - ], + "inputs": [], "outputs": [ { - "name": "IMAGE", - "type": "IMAGE", + "name": "face_toolkits", + "type": "FACE_TOOLKITS", "links": [ - 55 + 32, + 42, + 51 ], "slot_index": 0 } ], "properties": { - "Node name for S&R": "HMPipelineImage" + "Node name for S&R": "HMFaceToolkitsLoader" }, "widgets_values": [ - 0, - "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", - "", - 25, - 77251, - "randomize", - 2 + 0 ] }, { - "id": 37, + "id": 11, + "type": "GetReferenceImageRT", + "pos": { + "0": 749, + "1": 546 + }, + "size": { + "0": 241.79998779296875, + "1": 46 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 50 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 51 + } + ], + "outputs": [ + { + "name": "ref_rt", + "type": "REFRT", + "links": [ + 56 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetReferenceImageRT" + }, + "widgets_values": [] + }, + { + "id": 31, + "type": "GetVideoDriveParams", + "pos": { + "0": 447, + "1": 745 + }, + "size": { + "0": 330, + "1": 46 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 42 + }, + { + "name": "images", + "type": "IMAGE", + "link": 53 + } + ], + "outputs": [ + { + "name": "drive_params", + "type": "DRIVE_PARAMS", + "links": [ + 57 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetVideoDriveParams" + }, + "widgets_values": [] + }, + { + "id": 38, "type": "PreviewImage", "pos": { - "0": 1501, - "1": 377 + "0": 1483, + "1": 508 }, "size": [ 210, @@ -303,7 +316,7 @@ { "name": "images", "type": "IMAGE", - "link": 55 + "link": 58 } ], "outputs": [], @@ -312,87 +325,66 @@ } }, { - "id": 35, - "type": "HMImagePipelineLoader", + "id": 37, + "type": "HMPipelineImage", "pos": { - "0": 58, - "1": 113 + "0": 1083, + "1": 508 }, "size": { - "0": 352.79998779296875, - "1": 106 + "0": 315, + "1": 262 }, "flags": {}, - "order": 2, + "order": 8, "mode": 0, - "inputs": [], - "outputs": [ + "inputs": [ { "name": "hm_image_pipeline", "type": "HMIMAGEPIPELINE", - "links": [ - 53 - ], - "slot_index": 0 + "link": 54 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 55 + }, + { + "name": "ref_rt", + "type": "REFRT", + "link": 56 + }, + { + "name": "drive_params", + "type": "DRIVE_PARAMS", + "link": 57 } ], - "properties": { - "Node name for S&R": "HMImagePipelineLoader" - }, - "widgets_values": [ - "None", - "None", - 0 - ] - }, - { - "id": 36, - "type": "LoadImage", - "pos": { - "0": 60, - "1": 862 - }, - "size": [ - 315, - 314 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ - 54 + 58 ], "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null } ], "properties": { - "Node name for S&R": "LoadImage" + "Node name for S&R": "HMPipelineImage" }, "widgets_values": [ - "yao.jpg", - "image" + 0, + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 47116, + "randomize", + 2 ] } ], "links": [ - [ - 13, - 2, - 0, - 11, - 0, - "FACE_TOOLKITS" - ], [ 31, 8, @@ -417,14 +409,6 @@ 0, "IMAGE" ], - [ - 37, - 25, - 0, - 11, - 1, - "IMAGE" - ], [ 42, 2, @@ -435,38 +419,22 @@ ], [ 50, - 11, - 0, - 34, - 2, - "REFRT" - ], - [ - 51, - 31, - 0, - 34, - 3, - "DRIVE_PARAMS" - ], - [ - 52, 25, 0, - 34, - 1, + 11, + 0, "IMAGE" ], [ - 53, - 35, + 51, + 2, 0, - 34, - 0, - "HMIMAGEPIPELINE" + 11, + 1, + "FACE_TOOLKITS" ], [ - 54, + 53, 36, 0, 31, @@ -474,11 +442,43 @@ "IMAGE" ], [ - 55, - 34, + 54, + 35, 0, 37, 0, + "HMIMAGEPIPELINE" + ], + [ + 55, + 25, + 0, + 37, + 1, + "IMAGE" + ], + [ + 56, + 11, + 0, + 37, + 2, + "REFRT" + ], + [ + 57, + 31, + 0, + 37, + 3, + "DRIVE_PARAMS" + ], + [ + 58, + 37, + 0, + 38, + 0, "IMAGE" ] ], @@ -486,10 +486,10 @@ "config": {}, "extra": { "ds": { - "scale": 1, + "scale": 0.8264462809917354, "offset": [ - 265.78003215780683, - -30.4541044667626 + 484.16679729452585, + 79.12268905130395 ] } }, diff --git a/workflows/hellomeme_video_simple_workflow.json b/workflows/hellomeme_video_simple_workflow.json index 351b6f9..1e642d9 100644 --- a/workflows/hellomeme_video_simple_workflow.json +++ b/workflows/hellomeme_video_simple_workflow.json @@ -1,162 +1,7 @@ { - "last_node_id": 33, - "last_link_id": 49, + "last_node_id": 34, + "last_link_id": 52, "nodes": [ - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 61, - "1": 290 - }, - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 31 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, - { - "id": 20, - "type": "VHS_LoadVideoFFmpeg", - "pos": { - "0": 66, - "1": 835 - }, - "size": { - "0": 303.46875, - "1": 238 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null, - "shape": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": null, - "shape": 7 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 43 - ], - "slot_index": 0 - }, - { - "name": "mask", - "type": "MASK", - "links": null - }, - { - "name": "audio", - "type": "AUDIO", - "links": [ - 25 - ], - "slot_index": 2 - }, - { - "name": "video_info", - "type": "VHS_VIDEOINFO", - "links": null - } - ], - "properties": { - "Node name for S&R": "VHS_LoadVideoFFmpeg" - }, - "widgets_values": { - "video": "jgz.mp4", - "force_rate": 15, - "force_size": "Disabled", - "custom_width": 512, - "custom_height": 512, - "frame_load_cap": 0, - "start_time": 0, - "choose video to upload": "image", - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "force_rate": 15, - "frame_load_cap": 0, - "start_time": 0, - "filename": "jgz.mp4", - "type": "input", - "format": "video/mp4" - }, - "muted": false - } - } - }, - { - "id": 2, - "type": "HMFaceToolkitsLoader", - "pos": { - "0": 61, - "1": 697 - }, - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "links": [ - 13, - 32, - 42 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMFaceToolkitsLoader" - }, - "widgets_values": [ - 0 - ] - }, { "id": 25, "type": "CropReferenceImage", @@ -189,8 +34,8 @@ "type": "IMAGE", "links": [ 34, - 37, - 46 + 46, + 50 ], "slot_index": 0 } @@ -227,47 +72,6 @@ }, "widgets_values": [] }, - { - "id": 11, - "type": "GetReferenceImageRT", - "pos": { - "0": 749, - "1": 546 - }, - "size": { - "0": 241.79998779296875, - "1": 46 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 13 - }, - { - "name": "image", - "type": "IMAGE", - "link": 37 - } - ], - "outputs": [ - { - "name": "ref_rt", - "type": "REFRT", - "links": [ - 47 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetReferenceImageRT" - }, - "widgets_values": [] - }, { "id": 32, "type": "HMPipelineVideo", @@ -286,7 +90,7 @@ { "name": "hm_video_pipeline", "type": "HMVIDEOPIPELINE", - "link": 49 + "link": 52 }, { "name": "ref_image", @@ -322,7 +126,7 @@ "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", "", 25, - 28524, + 13113, "randomize", 2 ] @@ -431,7 +235,7 @@ "hidden": false, "paused": false, "params": { - "filename": "HelloMeme_00068-audio.mp4", + "filename": "HelloMeme_00073-audio.mp4", "subfolder": "", "type": "output", "format": "video/h264-mp4", @@ -442,11 +246,207 @@ } }, { - "id": 33, + "id": 8, + "type": "LoadImage", + "pos": { + "0": 61, + "1": 290 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 31 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 11, + "type": "GetReferenceImageRT", + "pos": { + "0": 749, + "1": 546 + }, + "size": { + "0": 241.79998779296875, + "1": 46 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 50 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 51 + } + ], + "outputs": [ + { + "name": "ref_rt", + "type": "REFRT", + "links": [ + 47 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetReferenceImageRT" + }, + "widgets_values": [] + }, + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": { + "0": 61, + "1": 697 + }, + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "links": [ + 32, + 42, + 51 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0 + ] + }, + { + "id": 20, + "type": "VHS_LoadVideoFFmpeg", + "pos": { + "0": 66, + "1": 835 + }, + "size": [ + 303.46875, + 535.46875 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 43 + ], + "slot_index": 0 + }, + { + "name": "mask", + "type": "MASK", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 25 + ], + "slot_index": 2 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoFFmpeg" + }, + "widgets_values": { + "video": "jgz.mp4", + "force_rate": 15, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "start_time": 0, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "force_rate": 15, + "frame_load_cap": 0, + "start_time": 0, + "filename": "jgz.mp4", + "type": "input", + "format": "video/mp4" + }, + "muted": false + } + } + }, + { + "id": 34, "type": "HMVideoSimplePipelineLoader", "pos": { - "0": 59, - "1": 118 + "0": 56, + "1": 85 }, "size": { "0": 428.4000244140625, @@ -461,7 +461,7 @@ "name": "hm_video_pipeline", "type": "HMVIDEOPIPELINE", "links": [ - 49 + 52 ], "slot_index": 0 } @@ -470,21 +470,13 @@ "Node name for S&R": "HMVideoSimplePipelineLoader" }, "widgets_values": [ - "None", + "SD1.5", "None", 0 ] } ], "links": [ - [ - 13, - 2, - 0, - 11, - 0, - "FACE_TOOLKITS" - ], [ 25, 20, @@ -517,14 +509,6 @@ 0, "IMAGE" ], - [ - 37, - 25, - 0, - 11, - 1, - "IMAGE" - ], [ 42, 2, @@ -574,8 +558,24 @@ "DRIVE_PARAMS" ], [ - 49, - 33, + 50, + 25, + 0, + 11, + 0, + "IMAGE" + ], + [ + 51, + 2, + 0, + 11, + 1, + "FACE_TOOLKITS" + ], + [ + 52, + 34, 0, 32, 0, @@ -586,10 +586,10 @@ "config": {}, "extra": { "ds": { - "scale": 1, + "scale": 0.8264462809917354, "offset": [ - 265.78003215780683, - -30.4541044667626 + 483.3601798628852, + 79.12268905130395 ] } }, diff --git a/workflows/hellomeme_video_workflow.json b/workflows/hellomeme_video_workflow.json index 6878b5c..0e7499c 100644 --- a/workflows/hellomeme_video_workflow.json +++ b/workflows/hellomeme_video_workflow.json @@ -1,196 +1,7 @@ { - "last_node_id": 32, - "last_link_id": 48, + "last_node_id": 33, + "last_link_id": 51, "nodes": [ - { - "id": 30, - "type": "HMVideoPipelineLoader", - "pos": { - "0": 55, - "1": 112 - }, - "size": { - "0": 352.79998779296875, - "1": 106 - }, - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "hm_video_pipeline", - "type": "HMVIDEOPIPELINE", - "links": [ - 45 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMVideoPipelineLoader" - }, - "widgets_values": [ - "None", - "None", - 0 - ] - }, - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 61, - "1": 290 - }, - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 31 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, - { - "id": 20, - "type": "VHS_LoadVideoFFmpeg", - "pos": { - "0": 66, - "1": 835 - }, - "size": { - "0": 303.46875, - "1": 238 - }, - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null, - "shape": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": null, - "shape": 7 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 43 - ], - "slot_index": 0 - }, - { - "name": "mask", - "type": "MASK", - "links": null - }, - { - "name": "audio", - "type": "AUDIO", - "links": [ - 25 - ], - "slot_index": 2 - }, - { - "name": "video_info", - "type": "VHS_VIDEOINFO", - "links": null - } - ], - "properties": { - "Node name for S&R": "VHS_LoadVideoFFmpeg" - }, - "widgets_values": { - "video": "jgz.mp4", - "force_rate": 15, - "force_size": "Disabled", - "custom_width": 512, - "custom_height": 512, - "frame_load_cap": 0, - "start_time": 0, - "choose video to upload": "image", - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "force_rate": 15, - "frame_load_cap": 0, - "start_time": 0, - "filename": "jgz.mp4", - "type": "input", - "format": "video/mp4" - }, - "muted": false - } - } - }, - { - "id": 2, - "type": "HMFaceToolkitsLoader", - "pos": { - "0": 61, - "1": 697 - }, - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "links": [ - 13, - 32, - 42 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMFaceToolkitsLoader" - }, - "widgets_values": [ - 0 - ] - }, { "id": 25, "type": "CropReferenceImage", @@ -223,8 +34,8 @@ "type": "IMAGE", "links": [ 34, - 37, - 46 + 46, + 50 ], "slot_index": 0 } @@ -261,47 +72,6 @@ }, "widgets_values": [] }, - { - "id": 11, - "type": "GetReferenceImageRT", - "pos": { - "0": 749, - "1": 546 - }, - "size": { - "0": 241.79998779296875, - "1": 46 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 13 - }, - { - "name": "image", - "type": "IMAGE", - "link": 37 - } - ], - "outputs": [ - { - "name": "ref_rt", - "type": "REFRT", - "links": [ - 47 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetReferenceImageRT" - }, - "widgets_values": [] - }, { "id": 32, "type": "HMPipelineVideo", @@ -320,7 +90,7 @@ { "name": "hm_video_pipeline", "type": "HMVIDEOPIPELINE", - "link": 45 + "link": 49 }, { "name": "ref_image", @@ -356,7 +126,7 @@ "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", "", 25, - 89993, + 32295, "randomize", 2 ] @@ -474,17 +244,240 @@ "muted": false } } + }, + { + "id": 8, + "type": "LoadImage", + "pos": { + "0": 61, + "1": 290 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 31 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 33, + "type": "HMVideoPipelineLoader", + "pos": { + "0": 66, + "1": 85 + }, + "size": { + "0": 352.79998779296875, + "1": 130 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "links": [ + 49 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMVideoPipelineLoader" + }, + "widgets_values": [ + "SD1.5", + "None", + 12, + 0 + ] + }, + { + "id": 11, + "type": "GetReferenceImageRT", + "pos": { + "0": 749, + "1": 546 + }, + "size": { + "0": 241.79998779296875, + "1": 46 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 50 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 51 + } + ], + "outputs": [ + { + "name": "ref_rt", + "type": "REFRT", + "links": [ + 47 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetReferenceImageRT" + }, + "widgets_values": [] + }, + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": { + "0": 61, + "1": 697 + }, + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "links": [ + 32, + 42, + 51 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0 + ] + }, + { + "id": 20, + "type": "VHS_LoadVideoFFmpeg", + "pos": { + "0": 66, + "1": 835 + }, + "size": [ + 303.46875, + 535.46875 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 43 + ], + "slot_index": 0 + }, + { + "name": "mask", + "type": "MASK", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 25 + ], + "slot_index": 2 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoFFmpeg" + }, + "widgets_values": { + "video": "jgz.mp4", + "force_rate": 15, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "start_time": 0, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "force_rate": 15, + "frame_load_cap": 0, + "start_time": 0, + "filename": "jgz.mp4", + "type": "input", + "format": "video/mp4" + }, + "muted": false + } + } } ], "links": [ - [ - 13, - 2, - 0, - 11, - 0, - "FACE_TOOLKITS" - ], [ 25, 20, @@ -517,14 +510,6 @@ 0, "IMAGE" ], - [ - 37, - 25, - 0, - 11, - 1, - "IMAGE" - ], [ 42, 2, @@ -549,14 +534,6 @@ 0, "IMAGE" ], - [ - 45, - 30, - 0, - 32, - 0, - "HMVIDEOPIPELINE" - ], [ 46, 25, @@ -580,16 +557,40 @@ 32, 3, "DRIVE_PARAMS" + ], + [ + 49, + 33, + 0, + 32, + 0, + "HMVIDEOPIPELINE" + ], + [ + 50, + 25, + 0, + 11, + 0, + "IMAGE" + ], + [ + 51, + 2, + 0, + 11, + 1, + "FACE_TOOLKITS" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 1, + "scale": 0.8264462809917354, "offset": [ - 265.78003215780683, - -30.4541044667626 + 482.55341472616647, + 79.12268905130395 ] } },