diff --git a/example_workflows/hellomeme_image_workflow.json b/example_workflows/hellomeme_image_workflow.json deleted file mode 100644 index 0a3ebc3..0000000 --- a/example_workflows/hellomeme_image_workflow.json +++ /dev/null @@ -1 +0,0 @@ -{"last_node_id":63,"last_link_id":129,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,127],"slot_index":0,"localized_name":"face_toolkits"}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128,"localized_name":"image"},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57,"localized_name":"face_toolkits"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0,"localized_name":"IMAGE"}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":115,"localized_name":"images"}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,126],"slot_index":0,"localized_name":"landmarks"}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":116,"localized_name":"images"},{"name":"landmarks","type":"FACELANDMARKS222","link":64,"localized_name":"landmarks"}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0,"localized_name":"drive_pose"}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,125],"slot_index":0,"localized_name":"IMAGE"},{"name":"MASK","type":"MASK","links":null,"localized_name":"MASK"}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129,"localized_name":"hm_image_pipeline"},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119,"localized_name":"face_toolkits"},{"name":"ref_image","type":"IMAGE","link":118,"localized_name":"ref_image"},{"name":"drive_pose","type":"DRIVE_POSE","link":120,"localized_name":"drive_pose"},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":124,"shape":7,"localized_name":"drive_exp"},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":null,"shape":7,"localized_name":"drive_exp2"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0,"localized_name":"IMAGE"},{"name":"LATENT","type":"LATENT","links":null,"localized_name":"LATENT"}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,68992207739336,"randomize",2,0]},{"id":61,"type":"GetDriveExpression","pos":[857,823],"size":[292.20001220703125,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":127,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":125,"localized_name":"images"},{"name":"landmarks","type":"FACELANDMARKS222","link":126,"localized_name":"landmarks"}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[124],"slot_index":0,"localized_name":"drive_exp"}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0,"localized_name":"IMAGE"},{"name":"MASK","type":"MASK","links":null,"localized_name":"MASK"}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[210,246],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122,"localized_name":"images"}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0,"localized_name":"hm_image_pipeline"}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v3","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[124,61,0,58,4,"DRIVE_EXPRESSION"],[125,57,0,61,1,"IMAGE"],[126,43,0,61,2,"FACELANDMARKS222"],[127,2,0,61,0,"FACE_TOOLKITS"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file diff --git a/example_workflows/hellomeme_video_v2_workflow.json b/example_workflows/hellomeme_video_v2_workflow.json deleted file mode 100644 index ee35b0a..0000000 --- a/example_workflows/hellomeme_video_v2_workflow.json +++ /dev/null @@ -1 +0,0 @@ -{"last_node_id":59,"last_link_id":118,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,114],"slot_index":0,"localized_name":"face_toolkits"}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":116,"localized_name":"image"},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57,"localized_name":"face_toolkits"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[99],"slot_index":0,"localized_name":"IMAGE"}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":82,"localized_name":"images"}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,113],"slot_index":0,"localized_name":"landmarks"}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":83,"localized_name":"images"},{"name":"landmarks","type":"FACELANDMARKS222","link":64,"localized_name":"landmarks"}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0,"localized_name":"drive_pose"}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115,"localized_name":"hm_video_pipeline"},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":97,"localized_name":"face_toolkits"},{"name":"ref_image","type":"IMAGE","link":99,"localized_name":"ref_image"},{"name":"drive_pose","type":"DRIVE_POSE","link":100,"localized_name":"drive_pose"},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":null,"shape":7,"localized_name":"drive_exp"},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":111,"shape":7,"localized_name":"drive_exp2"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[117],"slot_index":0,"localized_name":"IMAGE"},{"name":"LATENT","type":"LATENT","links":null,"localized_name":"LATENT"}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,293728504444278,"randomize",2,0]},{"id":56,"type":"GetDriveExpression2","pos":[844,816],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":114,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":112,"localized_name":"images"},{"name":"landmarks","type":"FACELANDMARKS222","link":113,"localized_name":"landmarks"}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[111],"slot_index":0,"localized_name":"drive_exp2"}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":59,"type":"VHS_VideoCombine","pos":[1668.69091796875,739.6614379882812],"size":[329.22479248046875,633.2247924804688],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":117,"localized_name":"images"},{"name":"audio","type":"AUDIO","link":118,"shape":7,"localized_name":"audio"},{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7,"localized_name":"meta_batch"},{"name":"vae","type":"VAE","link":null,"shape":7,"localized_name":"vae"}],"outputs":[{"name":"Filenames","type":"VHS_FILENAMES","links":null,"localized_name":"Filenames"}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00408-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[116],"slot_index":0,"localized_name":"IMAGE"},{"name":"MASK","type":"MASK","links":null,"localized_name":"MASK"}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":48,"type":"VHS_LoadVideo","pos":[106,867],"size":[247.455078125,680.3645629882812],"flags":{},"order":3,"mode":0,"inputs":[{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7,"localized_name":"meta_batch"},{"name":"vae","type":"VAE","link":null,"shape":7,"localized_name":"vae"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[82,83,112],"slot_index":0,"localized_name":"IMAGE"},{"name":"frame_count","type":"INT","links":null,"localized_name":"frame_count"},{"name":"audio","type":"AUDIO","links":[118],"slot_index":2,"localized_name":"audio"},{"name":"video_info","type":"VHS_VIDEOINFO","links":null,"localized_name":"video_info"}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"tiktok.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"tiktok.mp4","type":"input","format":"video/mp4"},"muted":false}}},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0,"localized_name":"hm_video_pipeline"}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]MoreDetails","same as checkpoint","v3","x1","huggingface",1]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[82,48,0,43,1,"IMAGE"],[83,48,0,44,1,"IMAGE"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[111,56,0,53,5,"DRIVE_EXPRESSION2"],[112,48,0,56,1,"IMAGE"],[113,43,0,56,2,"FACELANDMARKS222"],[114,2,0,56,0,"FACE_TOOLKITS"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[117,53,0,59,0,"IMAGE"],[118,48,2,59,1,"AUDIO"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file diff --git a/example_workflows/hellomeme_video_workflow.json b/example_workflows/hellomeme_video_workflow.json deleted file mode 100644 index 2f183e8..0000000 --- a/example_workflows/hellomeme_video_workflow.json +++ /dev/null @@ -1 +0,0 @@ -{"last_node_id":60,"last_link_id":122,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,119],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":116},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[99],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":82}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,120],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":83},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":97},{"name":"ref_image","type":"IMAGE","link":99},{"name":"drive_pose","type":"DRIVE_POSE","link":100},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":122,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[117],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,331732715385901,"randomize",2,0]},{"id":60,"type":"GetDriveExpression","pos":[845,813],"size":[292.20001220703125,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"images","type":"IMAGE","link":121},{"name":"landmarks","type":"FACELANDMARKS222","link":120}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[122],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[116],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":59,"type":"VHS_VideoCombine","pos":[1619,800],"size":[280.4215087890625,584.4215087890625],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":117},{"name":"audio","type":"AUDIO","link":118,"shape":7},{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00409-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v3","x1","huggingface",1]},{"id":48,"type":"VHS_LoadVideo","pos":[106,867],"size":[247.455078125,680.3645629882812],"flags":{},"order":3,"mode":0,"inputs":[{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[82,83,121],"slot_index":0},{"name":"frame_count","type":"INT","links":null},{"name":"audio","type":"AUDIO","links":[118],"slot_index":2},{"name":"video_info","type":"VHS_VIDEOINFO","links":null}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"tiktok.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"tiktok.mp4","type":"input","format":"video/mp4"},"muted":false}}}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[82,48,0,43,1,"IMAGE"],[83,48,0,44,1,"IMAGE"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[117,53,0,59,0,"IMAGE"],[118,48,2,59,1,"AUDIO"],[119,2,0,60,0,"FACE_TOOLKITS"],[120,43,0,60,2,"FACELANDMARKS222"],[121,48,0,60,1,"IMAGE"],[122,60,0,53,4,"DRIVE_EXPRESSION"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015779,"offset":[634.1362099561715,-70.9744212566187]}},"version":0.4} \ No newline at end of file diff --git a/example_workflows/image_generation_ctrl1.jpg b/example_workflows/image_generation_ctrl1.jpg new file mode 100644 index 0000000..b7809ce Binary files /dev/null and b/example_workflows/image_generation_ctrl1.jpg differ diff --git a/example_workflows/image_generation_ctrl1.json b/example_workflows/image_generation_ctrl1.json new file mode 100644 index 0000000..f0597c4 --- /dev/null +++ b/example_workflows/image_generation_ctrl1.json @@ -0,0 +1 @@ +{"last_node_id":63,"last_link_id":129,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,127],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,126],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,125],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","shape":7,"link":124},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","shape":7,"link":null}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,859627586602455,"randomize",2,0]},{"id":61,"type":"GetDriveExpression","pos":[857,823],"size":[292.20001220703125,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":127},{"name":"images","type":"IMAGE","link":125},{"name":"landmarks","type":"FACELANDMARKS222","link":126}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[124],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[210,246],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v4","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[124,61,0,58,4,"DRIVE_EXPRESSION"],[125,57,0,61,1,"IMAGE"],[126,43,0,61,2,"FACELANDMARKS222"],[127,2,0,61,0,"FACE_TOOLKITS"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file diff --git a/example_workflows/image_generation_ctrl2.jpg b/example_workflows/image_generation_ctrl2.jpg new file mode 100644 index 0000000..a57e142 Binary files /dev/null and b/example_workflows/image_generation_ctrl2.jpg differ diff --git a/example_workflows/hellomeme_image_v2_workflow.json b/example_workflows/image_generation_ctrl2.json similarity index 79% rename from example_workflows/hellomeme_image_v2_workflow.json rename to example_workflows/image_generation_ctrl2.json index 0bc1797..da4fab7 100644 --- a/example_workflows/hellomeme_image_v2_workflow.json +++ b/example_workflows/image_generation_ctrl2.json @@ -1 +1 @@ -{"last_node_id":64,"last_link_id":133,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,132],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,130],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,131],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":null,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":133,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,293309029136420,"randomize",2,0]},{"id":64,"type":"GetDriveExpression2","pos":[850,827],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":132},{"name":"images","type":"IMAGE","link":131},{"name":"landmarks","type":"FACELANDMARKS222","link":130}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[133],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[290.7474365234375,283.26800537109375],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v3","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"],[130,43,0,64,2,"FACELANDMARKS222"],[131,57,0,64,1,"IMAGE"],[132,2,0,64,0,"FACE_TOOLKITS"],[133,64,0,58,5,"DRIVE_EXPRESSION2"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file +{"last_node_id":64,"last_link_id":133,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,132],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,130],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,131],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","shape":7,"link":null},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","shape":7,"link":133}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,404637369688717,"randomize",2,0]},{"id":64,"type":"GetDriveExpression2","pos":[850,827],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":132},{"name":"images","type":"IMAGE","link":131},{"name":"landmarks","type":"FACELANDMARKS222","link":130}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[133],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[290.7474365234375,283.26800537109375],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]MoreDetails","same as checkpoint","v4","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"],[130,43,0,64,2,"FACELANDMARKS222"],[131,57,0,64,1,"IMAGE"],[132,2,0,64,0,"FACE_TOOLKITS"],[133,64,0,58,5,"DRIVE_EXPRESSION2"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file diff --git a/example_workflows/video_generation_ctrl1.jpg b/example_workflows/video_generation_ctrl1.jpg new file mode 100644 index 0000000..3192669 Binary files /dev/null and b/example_workflows/video_generation_ctrl1.jpg differ diff --git a/example_workflows/video_generation_ctrl1.json b/example_workflows/video_generation_ctrl1.json new file mode 100644 index 0000000..299a1f6 --- /dev/null +++ b/example_workflows/video_generation_ctrl1.json @@ -0,0 +1,682 @@ +{ + "last_node_id": 60, + "last_link_id": 122, + "nodes": [ + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": [ + 43, + 699 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "links": [ + 57, + 62, + 65, + 97, + 119 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0, + "huggingface" + ] + }, + { + "id": 39, + "type": "CropPortrait", + "pos": [ + 516, + 537 + ], + "size": [ + 241.79998779296875, + 46 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 116 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 57 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 99 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CropPortrait" + }, + "widgets_values": [] + }, + { + "id": 43, + "type": "GetFaceLandmarks", + "pos": [ + 465, + 771 + ], + "size": [ + 292.20001220703125, + 46 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 62 + }, + { + "name": "images", + "type": "IMAGE", + "link": 82 + } + ], + "outputs": [ + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "links": [ + 64, + 120 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetFaceLandmarks" + }, + "widgets_values": [] + }, + { + "id": 44, + "type": "GetDrivePose", + "pos": [ + 842, + 677 + ], + "size": [ + 304.79998779296875, + 66 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 65 + }, + { + "name": "images", + "type": "IMAGE", + "link": 83 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 64 + } + ], + "outputs": [ + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "links": [ + 100 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDrivePose" + }, + "widgets_values": [] + }, + { + "id": 53, + "type": "HMPipelineVideo", + "pos": [ + 1258, + 562 + ], + "size": [ + 315, + 350 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "link": 115 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 97 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 99 + }, + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "link": 100 + }, + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "shape": 7, + "link": 122 + }, + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "shape": 7, + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 117 + ], + "slot_index": 0 + }, + { + "name": "LATENT", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "HMPipelineVideo" + }, + "widgets_values": [ + 0, + 4, + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 990809923057900, + "randomize", + 2, + 0 + ] + }, + { + "id": 60, + "type": "GetDriveExpression", + "pos": [ + 845, + 813 + ], + "size": [ + 292.20001220703125, + 66 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 119 + }, + { + "name": "images", + "type": "IMAGE", + "link": 121 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 120 + } + ], + "outputs": [ + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "links": [ + 122 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDriveExpression" + }, + "widgets_values": [] + }, + { + "id": 58, + "type": "LoadImage", + "pos": [ + 56, + 273 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 116 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "toon.png", + "image" + ] + }, + { + "id": 59, + "type": "VHS_VideoCombine", + "pos": [ + 1619, + 800 + ], + "size": [ + 280.4215087890625, + 584.4215087890625 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 117 + }, + { + "name": "audio", + "type": "AUDIO", + "shape": 7, + "link": 118 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "shape": 7, + "link": null + }, + { + "name": "vae", + "type": "VAE", + "shape": 7, + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 18, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00409-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + }, + "muted": false + } + } + }, + { + "id": 48, + "type": "VHS_LoadVideo", + "pos": [ + 106, + 867 + ], + "size": [ + 247.455078125, + 304 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "shape": 7, + "link": null + }, + { + "name": "vae", + "type": "VAE", + "shape": 7, + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82, + 83, + 121 + ], + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 118 + ], + "slot_index": 2 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "tiktok.mp4", + "force_rate": 15, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "force_rate": 15, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "filename": "tiktok.mp4", + "type": "input", + "format": "video/mp4" + }, + "muted": false + } + } + }, + { + "id": 57, + "type": "HMVideoPipelineLoader", + "pos": [ + 813, + 343 + ], + "size": [ + 352.79998779296875, + 202 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "links": [ + 115 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMVideoPipelineLoader" + }, + "widgets_values": [ + "[preset]DisneyPixarCartoonB", + "[preset]MoreDetails", + "same as checkpoint", + "v4", + "x1", + "huggingface", + 1 + ] + } + ], + "links": [ + [ + 57, + 2, + 0, + 39, + 1, + "FACE_TOOLKITS" + ], + [ + 62, + 2, + 0, + 43, + 0, + "FACE_TOOLKITS" + ], + [ + 64, + 43, + 0, + 44, + 2, + "FACELANDMARKS222" + ], + [ + 65, + 2, + 0, + 44, + 0, + "FACE_TOOLKITS" + ], + [ + 82, + 48, + 0, + 43, + 1, + "IMAGE" + ], + [ + 83, + 48, + 0, + 44, + 1, + "IMAGE" + ], + [ + 97, + 2, + 0, + 53, + 1, + "FACE_TOOLKITS" + ], + [ + 99, + 39, + 0, + 53, + 2, + "IMAGE" + ], + [ + 100, + 44, + 0, + 53, + 3, + "DRIVE_POSE" + ], + [ + 115, + 57, + 0, + 53, + 0, + "HMVIDEOPIPELINE" + ], + [ + 116, + 58, + 0, + 39, + 0, + "IMAGE" + ], + [ + 117, + 53, + 0, + 59, + 0, + "IMAGE" + ], + [ + 118, + 48, + 2, + 59, + 1, + "AUDIO" + ], + [ + 119, + 2, + 0, + 60, + 0, + "FACE_TOOLKITS" + ], + [ + 120, + 43, + 0, + 60, + 2, + "FACELANDMARKS222" + ], + [ + 121, + 48, + 0, + 60, + 1, + "IMAGE" + ], + [ + 122, + 60, + 0, + 53, + 4, + "DRIVE_EXPRESSION" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015779, + "offset": [ + 634.1362099561715, + -70.9744212566187 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/video_generation_ctrl2.jpg b/example_workflows/video_generation_ctrl2.jpg new file mode 100644 index 0000000..ac659bf Binary files /dev/null and b/example_workflows/video_generation_ctrl2.jpg differ diff --git a/example_workflows/video_generation_ctrl2.json b/example_workflows/video_generation_ctrl2.json new file mode 100644 index 0000000..b1b66c2 --- /dev/null +++ b/example_workflows/video_generation_ctrl2.json @@ -0,0 +1 @@ +{"last_node_id":59,"last_link_id":118,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,114],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"image","localized_name":"image","type":"IMAGE","link":116},{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","localized_name":"图像","type":"IMAGE","links":[99],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","localized_name":"images","type":"IMAGE","link":82}],"outputs":[{"name":"landmarks","localized_name":"landmarks","type":"FACELANDMARKS222","links":[64,113],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","localized_name":"images","type":"IMAGE","link":83},{"name":"landmarks","localized_name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","localized_name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_video_pipeline","localized_name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115},{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":97},{"name":"ref_image","localized_name":"ref_image","type":"IMAGE","link":99},{"name":"drive_pose","localized_name":"drive_pose","type":"DRIVE_POSE","link":100},{"name":"drive_exp","localized_name":"drive_exp","type":"DRIVE_EXPRESSION","shape":7,"link":null},{"name":"drive_exp2","localized_name":"drive_exp2","type":"DRIVE_EXPRESSION2","shape":7,"link":111}],"outputs":[{"name":"IMAGE","localized_name":"图像","type":"IMAGE","links":[117],"slot_index":0},{"name":"LATENT","localized_name":"Latent","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,293728504444278,"randomize",2,0]},{"id":56,"type":"GetDriveExpression2","pos":[844,816],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":114},{"name":"images","localized_name":"images","type":"IMAGE","link":112},{"name":"landmarks","localized_name":"landmarks","type":"FACELANDMARKS222","link":113}],"outputs":[{"name":"drive_exp2","localized_name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[111],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":59,"type":"VHS_VideoCombine","pos":[1668.69091796875,739.6614379882812],"size":[329.22479248046875,633.2247924804688],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","localized_name":"images","type":"IMAGE","link":117},{"name":"audio","localized_name":"audio","type":"AUDIO","shape":7,"link":118},{"name":"meta_batch","localized_name":"meta_batch","type":"VHS_BatchManager","shape":7,"link":null},{"name":"vae","localized_name":"vae","type":"VAE","shape":7,"link":null}],"outputs":[{"name":"Filenames","localized_name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00408-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","localized_name":"图像","type":"IMAGE","links":[116],"slot_index":0},{"name":"MASK","localized_name":"遮罩","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":48,"type":"VHS_LoadVideo","pos":[106,867],"size":[247.455078125,304],"flags":{},"order":2,"mode":0,"inputs":[{"name":"meta_batch","localized_name":"meta_batch","type":"VHS_BatchManager","shape":7,"link":null},{"name":"vae","localized_name":"vae","type":"VAE","shape":7,"link":null}],"outputs":[{"name":"IMAGE","localized_name":"图像","type":"IMAGE","links":[82,83,112],"slot_index":0},{"name":"frame_count","localized_name":"frame_count","type":"INT","links":null},{"name":"audio","localized_name":"audio","type":"AUDIO","links":[118],"slot_index":2},{"name":"video_info","localized_name":"video_info","type":"VHS_VIDEOINFO","links":null}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"tiktok.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"tiktok.mp4","type":"input","format":"video/mp4"},"muted":false}}},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","localized_name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]MoreDetails","same as checkpoint","v4","x1","huggingface",1]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[82,48,0,43,1,"IMAGE"],[83,48,0,44,1,"IMAGE"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[111,56,0,53,5,"DRIVE_EXPRESSION2"],[112,48,0,56,1,"IMAGE"],[113,43,0,56,2,"FACELANDMARKS222"],[114,2,0,56,0,"FACE_TOOLKITS"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[117,53,0,59,0,"IMAGE"],[118,48,2,59,1,"AUDIO"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file diff --git a/hellomeme/models/__init__.py b/hellomeme/models/__init__.py index 608997c..5f06d5e 100644 --- a/hellomeme/models/__init__.py +++ b/hellomeme/models/__init__.py @@ -9,7 +9,7 @@ """ from .hm_denoising_motion import HMDenoisingMotion -from .hm_control import HMControlNet, HMControlNet2, HMV2ControlNet, HMV2ControlNet2, HMV3ControlNet +from .hm_control import HMControlNet, HMControlNet2, HMV2ControlNet, HMV2ControlNet2, HMV3ControlNet, HMControlNetBase, HM4SD15ControlProj from .hm_blocks import HMReferenceAdapter, HM3ReferenceAdapter, HM3MotionAdapter, HMPipeline from .hm_denoising_3d import HMDenoising3D from .hm3_denoising_3d import HM3Denoising3D diff --git a/hellomeme/models/hm_blocks.py b/hellomeme/models/hm_blocks.py index a9d7081..f11eabd 100644 --- a/hellomeme/models/hm_blocks.py +++ b/hellomeme/models/hm_blocks.py @@ -270,6 +270,158 @@ class SKReferenceAttentionV3(nn.Module): return hidden_states + self.proj(res2.contiguous()) + +class SmallUnet(nn.Module): + def __init__(self, in_channels: int = 3, + out_channels: int = 16, + cross_attention_dim: int = 1024, + mid_channels: Tuple[int] = (320, 640, 1280), + temporal_attn: bool = False): + super(SmallUnet, self).__init__() + + down_channels = [in_channels] + mid_channels + up_channels = mid_channels[::-1] + [out_channels] + self.down_blocks = nn.ModuleList([ + BasicBlock( + inplanes=down_channels[i-1], + planes=down_channels[i], + stride=2, + downsample=nn.Sequential( + nn.Conv2d(down_channels[i-1], down_channels[i], kernel_size=1, stride=2, bias=False), + nn.InstanceNorm2d(down_channels[i]), + nn.SiLU(), + ), + norm_layer=nn.InstanceNorm2d, + ) for i in range(1, len(down_channels)) + ]) + self.up_blocks = nn.ModuleList([ + nn.Sequential( + nn.Conv2d(up_channels[i-1], up_channels[i], kernel_size=3, padding=1), + nn.InstanceNorm2d(up_channels[i]), + nn.SiLU(), + nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False), + ) for i in range(1, len(up_channels)) + ]) + self.attn_blocks = nn.ModuleList([ + STKCrossAttention( + channel_in=c, + channel_mid=c, + cross_attention_dim=cross_attention_dim, + num_positional_embeddings=512, + num_positional_embeddings_hidden=1024, + temporal_attn=temporal_attn, + ) for c in mid_channels + ]) + + def forward(self, x, condition): + f = x.size(2) + x = rearrange(x, "b c f h w -> (b f) c h w") + skips = [] + ret_dict = {} + for down_block, attn_block in zip(self.down_blocks, self.attn_blocks): + x = rearrange(down_block(x), "(b f) c h w -> b f c h w", f=f) + x = attn_block(x, condition) + x = rearrange(x, "b f c h w -> (b f) c h w") + skips.append(x) + skips = skips[::-1][1:] + ret_dict['feat_0'] = rearrange(x, "(b f) c h w -> b c f h w", f=f) + for i, block in enumerate(self.up_blocks[:-1]): + x = block(x) + skips[i] + ret_dict[f'feat_{i+1}'] = rearrange(x, "(b f) c h w -> b c f h w", f=f) + x = self.up_blocks[-1](x) + ret_dict[f'feat_{i+2}'] = rearrange(x, "(b f) c h w -> b c f h w", f=f) + return ret_dict + + +class STKCrossAttention(nn.Module): + def __init__( + self, + channel_in: int, + channel_mid: int, + heads: int = 8, + cross_attention_dim: int = 320, + norm_elementwise_affine: bool = True, + norm_eps: float = 1e-5, + num_positional_embeddings: int = 64, + num_positional_embeddings_hidden: int = 64, + temporal_attn: bool = False, + ): + super().__init__() + + self.conv_in = nn.Conv2d(channel_in, channel_mid, kernel_size=3, padding=1) + self.pos_embed = SinusoidalPositionalEmbedding(channel_mid, max_seq_length=num_positional_embeddings) + self.pos_embed_hidden = SinusoidalPositionalEmbedding(cross_attention_dim, max_seq_length=num_positional_embeddings_hidden) + + self.norm = nn.LayerNorm(channel_mid, elementwise_affine=norm_elementwise_affine, eps=norm_eps) + self.attn1 = Attention( + query_dim=channel_mid, + heads=heads, + dim_head=channel_mid // heads, + dropout=0.0, + bias=False, + cross_attention_dim=cross_attention_dim, + upcast_attention=False, + out_bias=True, + ) + + self.attn2 = Attention( + query_dim=channel_mid, + heads=heads, + dim_head=channel_mid // heads, + dropout=0.0, + bias=False, + cross_attention_dim=cross_attention_dim, + upcast_attention=False, + out_bias=True, + ) + + if temporal_attn: + self.attn3 = Attention( + query_dim=channel_mid, + heads=heads, + dim_head=channel_mid // heads, + dropout=0.0, + bias=False, + cross_attention_dim=cross_attention_dim, + upcast_attention=False, + out_bias=True, + ) + + self.ff = FeedForward( + channel_mid, + dropout=0.0, + activation_fn="geglu", + final_dropout=False, + inner_dim=channel_mid*16, + bias=True, + ) + + self.proj = nn.Conv2d(channel_mid, channel_in, kernel_size=3, padding=1) + + def forward(self, input, hidden_stats): + b, f, _, h, w = input.shape + x = rearrange(input, "b f c h w -> (b f) c h w") + x = self.conv_in(x) + + hidden_stats = rearrange(hidden_stats, "b f c d -> (b f) c d") + x = rearrange(x, "b c h w -> (b h) w c") + x = self.attn1(self.norm(self.pos_embed(x)), + self.pos_embed_hidden(hidden_stats.repeat_interleave(h, dim=0))) + x = rearrange(x, "(b h) w c -> (b w) h c", h=h) + x = self.attn2(self.norm(self.pos_embed(x)), + self.pos_embed_hidden(hidden_stats.repeat_interleave(w, dim=0))) + if hasattr(self, 'attn3'): + x = rearrange(x, "(b f w) h c -> (b w h) f c", w=w, f=f) + x = self.attn3(self.norm(self.pos_embed(x)), + self.pos_embed_hidden(hidden_stats.repeat_interleave(w*h, dim=0))) + x = rearrange(x, "(b w h) f c -> (b f) (h w) c", w=w, h=h) + else: + x = rearrange(x, "(b f w) h c -> (b f) (h w) c", w=w, f=f) + x = self.norm(self.ff(x)) + x = rearrange(x, "(b f) (h w) c -> (b f) c h w", w=w, f=f) + x = self.proj(x) + return rearrange(x, "(b f) c h w -> b f c h w", f=f) + class SKCrossAttention(nn.Module): def __init__( self, diff --git a/hellomeme/models/hm_control.py b/hellomeme/models/hm_control.py index 4259524..c647e6f 100644 --- a/hellomeme/models/hm_control.py +++ b/hellomeme/models/hm_control.py @@ -17,8 +17,114 @@ from diffusers.models.modeling_utils import ModelMixin from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.models.embeddings import TimestepEmbedding, Timesteps from einops import rearrange -from .hm_blocks import SKCrossAttention +from .hm_blocks import SKCrossAttention, SmallUnet +class HMControlNetBase(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + in_channels: int = 3, + cross_attention_dim: int = 2048, + ): + super().__init__() + + self.emo_embedding = Timesteps(1024, True, 0) + self.emo_pre_proj = TimestepEmbedding(548, 16) + self.emo_proj = TimestepEmbedding(1024, cross_attention_dim) + + self.exp_embedding = Timesteps(1024, True, 0) + self.exp_pre_proj = TimestepEmbedding(51, 16) + self.face_pre_proj = TimestepEmbedding(3, 16) + self.exp_proj = TimestepEmbedding(1024, cross_attention_dim) + self.face_proj = TimestepEmbedding(1024, cross_attention_dim) + + self.ctrl_attn = SmallUnet(in_channels=in_channels, out_channels=320, + mid_channels=[320, 640, 1280], + cross_attention_dim=cross_attention_dim, + temporal_attn=False) + + def forward(self, condition, drive_coeff=None, face_parts=None, emo_embedding=None): + bs, video_length = condition.size(0), condition.size(2) + condition = rearrange(condition, "b c f h w -> (b f) c h w") + condition = F.interpolate(condition, size=(condition.size(2)//8, condition.size(3)//8), mode='bilinear', align_corners=False) + condition = rearrange(condition, "(b f) c h w -> b c f h w", f=video_length) + + if drive_coeff is None: + emo_coeff = rearrange(emo_embedding * 20., "b f c -> (b f c)") + emo_embedding = self.emo_embedding(emo_coeff) + emo_embedding = rearrange(emo_embedding, "(b f c) d -> (b f) d c", b=bs, f=video_length) + emo_embedding = self.emo_pre_proj(emo_embedding.to(dtype=condition.dtype)) + emo_embedding = rearrange(emo_embedding, "(b f) d c -> (b f) c d", f=video_length) + drive_embedding = self.emo_proj(emo_embedding) + else: + drive_coeff = rearrange(drive_coeff * 500., "b f c -> (b f c)") + exp_embedding = self.exp_embedding(drive_coeff) + exp_embedding = rearrange(exp_embedding, "(b f c) d -> (b f) d c", b=bs, f=video_length) + exp_embedding = self.exp_pre_proj(exp_embedding.to(dtype=condition.dtype)) + exp_embedding = rearrange(exp_embedding, "(b f) d c -> (b f) c d", f=video_length) + + face_parts = rearrange(face_parts, "b f c d -> (b f) d c") + face_embedding = self.face_pre_proj(face_parts) + face_embedding = rearrange(face_embedding, "(b f) d c -> (b f) c d", f=video_length) + + drive_embedding = self.exp_proj(exp_embedding) + self.face_proj(face_embedding) + + drive_embedding = rearrange(drive_embedding, "(b f) c d -> b f c d", f=video_length) + return self.ctrl_attn(condition, drive_embedding) + + +class HM4SD15ControlProj(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + ): + super().__init__() + self.map640_to_640 = nn.Conv2d(640, 640, kernel_size=3, padding=1, bias=False) + self.map640_to_1280 = nn.Conv2d(640, 1280, kernel_size=3, padding=1, bias=False) + self.map320_to_320 = nn.Conv2d(320, 320, kernel_size=3, padding=1, bias=False) + self.map320_to_640 = nn.Conv2d(320, 640, kernel_size=3, padding=1, bias=False) + self.map320_to_1280 = nn.Conv2d(320, 1280, kernel_size=3, padding=1, bias=False) + self.map1280_to_1280 = nn.Conv2d(1280, 1280, kernel_size=3, padding=1, bias=False) + self.map1280_to_1280_2 = nn.Conv2d(1280, 1280, kernel_size=3, padding=1, bias=False) + self.map1280_to_1280_3 = nn.Conv2d(1280, 1280, kernel_size=3, padding=1, bias=False) + + def forward(self, control_dict): + video_length = control_dict['feat_0'].size(2) + ret_dict = dict( + up3_0=rearrange( + self.map1280_to_1280( + rearrange(control_dict['feat_0'], "b c f h w -> (b f) c h w"), + ), "(b f) c h w -> b c f h w", f=video_length), + up3_1=rearrange( + self.map640_to_1280( + rearrange(control_dict['feat_1'], "b c f h w -> (b f) c h w"), + ), "(b f) c h w -> b c f h w", f=video_length), + up3_2=rearrange( + self.map320_to_1280( + rearrange(control_dict['feat_2'], "b c f h w -> (b f) c h w"), + ), "(b f) c h w -> b c f h w", f=video_length), + up3_3=rearrange( + self.map320_to_640( + rearrange(control_dict['feat_3'], "b c f h w -> (b f) c h w"), + ), "(b f) c h w -> b c f h w", f=video_length), + down3_0=rearrange( + self.map320_to_320( + rearrange(control_dict['feat_2'], "b c f h w -> (b f) c h w"), + ), "(b f) c h w -> b c f h w", f=video_length), + down3_1=rearrange( + self.map640_to_640( + rearrange(control_dict['feat_1'], "b c f h w -> (b f) c h w"), + ), "(b f) c h w -> b c f h w", f=video_length), + down3_2=rearrange( + self.map1280_to_1280_2( + rearrange(control_dict['feat_0'], "b c f h w -> (b f) c h w"), + ), "(b f) c h w -> b c f h w", f=video_length), + down3_3=rearrange( + self.map1280_to_1280_3( + rearrange(control_dict['feat_0'], "b c f h w -> (b f) c h w"), + ), "(b f) c h w -> b c f h w", f=video_length), + ) + return ret_dict class HMControlNet(ModelMixin, ConfigMixin): @register_to_config diff --git a/hellomeme/pipelines/pipline_hm3_image.py b/hellomeme/pipelines/pipline_hm3_image.py index 0c62a5f..8b30086 100644 --- a/hellomeme/pipelines/pipline_hm3_image.py +++ b/hellomeme/pipelines/pipline_hm3_image.py @@ -20,9 +20,7 @@ from diffusers.utils.torch_utils import randn_tensor from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput from diffusers import DPMSolverMultistepScheduler from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents - -from ..models import HM3Denoising3D, HMV3ControlNet, HMPipeline, HM3ReferenceAdapter - +from ..models import HM3Denoising3D, HMV3ControlNet, HMPipeline, HM3ReferenceAdapter, HMControlNetBase, HM4SD15ControlProj class HM3ImagePipeline(HMPipeline): def caryomitosis(self, **kwargs): @@ -45,14 +43,24 @@ class HM3ImagePipeline(HMPipeline): self.safety_checker.cpu() def insert_hm_modules(self, version='v3', dtype=torch.float16, modelscope=False): - + self.version = version if modelscope: from modelscope import snapshot_download - hm_reference_dir = snapshot_download('songkey/hm3_reference') - hm_control_dir = snapshot_download('songkey/hm3_control_mix') + if version == 'v3': + hm_reference_dir = snapshot_download('songkey/hm3_reference') + hm_control_dir = snapshot_download('songkey/hm3_control_mix') + else: + hm_reference_dir = snapshot_download('songkey/hm4_reference') + hm_control_dir = snapshot_download('songkey/hm_control_base') + hm_control_proj_dir = snapshot_download('songkey/hm4_control_proj') else: - hm_reference_dir = 'songkey/hm3_reference' - hm_control_dir = 'songkey/hm3_control_mix' + if version == 'v3': + hm_reference_dir = 'songkey/hm3_reference' + hm_control_dir = 'songkey/hm3_control_mix' + else: + hm_reference_dir = 'songkey/hm4_reference' + hm_control_dir = 'songkey/hm_control_base' + hm_control_proj_dir = 'songkey/hm4_control_proj' if isinstance(self.unet, HM3Denoising3D): hm_adapter = HM3ReferenceAdapter.from_pretrained(hm_reference_dir) @@ -65,7 +73,17 @@ class HM3ImagePipeline(HMPipeline): if hasattr(self, "mp_control"): del self.mp_control - self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir) + if hasattr(self, "mp_control_proj"): + del self.mp_control_proj + + if version == 'v3': + self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir) + else: + self.mp_control = HMControlNetBase.from_pretrained(hm_control_dir) + self.mp_control_proj = HM4SD15ControlProj.from_pretrained(hm_control_proj_dir) + + self.mp_control_proj.to(device='cpu', dtype=dtype).eval() + self.mp_control.to(device='cpu', dtype=dtype).eval() self.vae.to(device='cpu', dtype=dtype).eval() @@ -233,6 +251,8 @@ class HM3ImagePipeline(HMPipeline): control_latents = {} self.mp_control.to(device=device) + if hasattr(self, 'mp_control_proj') and self.version == 'v4': + self.mp_control_proj.to(device=device) if 'drive_coeff' in drive_params: drive_coeff = drive_params['drive_coeff'].clone().to(device=device) face_parts = drive_params['face_parts'].clone().to(device=device) @@ -240,14 +260,20 @@ class HM3ImagePipeline(HMPipeline): drive_coeff = torch.cat([torch.zeros_like(drive_coeff), drive_coeff], dim=0) face_parts = torch.cat([torch.zeros_like(face_parts), face_parts], dim=0) control_latents1 = self.mp_control(condition=condition, drive_coeff=drive_coeff, face_parts=face_parts) + if self.version == 'v4': + control_latents1 = self.mp_control_proj(control_latents1) control_latents.update(control_latents1) elif 'pd_fpg' in drive_params: pd_fpg = drive_params['pd_fpg'].clone().to(device=device) if self.do_classifier_free_guidance: pd_fpg = torch.cat([torch.zeros_like(pd_fpg), pd_fpg], dim=0) control_latents2 = self.mp_control(condition=condition, emo_embedding=pd_fpg) + if self.version == 'v4': + control_latents2 = self.mp_control_proj(control_latents2) control_latents.update(control_latents2) self.mp_control.cpu() + if self.version == 'v4': + self.mp_control_proj.cpu() # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) diff --git a/hellomeme/pipelines/pipline_hm3_video.py b/hellomeme/pipelines/pipline_hm3_video.py index 57cb58b..73abd1f 100644 --- a/hellomeme/pipelines/pipline_hm3_video.py +++ b/hellomeme/pipelines/pipline_hm3_video.py @@ -24,7 +24,7 @@ from diffusers import MotionAdapter, DPMSolverMultistepScheduler from diffusers.utils.torch_utils import randn_tensor from ..models import (HM3Denoising3D, HM3DenoisingMotion, HM3MotionAdapter, - HMV3ControlNet, HM3ReferenceAdapter, HMPipeline) + HMV3ControlNet, HM3ReferenceAdapter, HMPipeline, HMControlNetBase, HM4SD15ControlProj) class HM3VideoPipeline(HMPipeline): def caryomitosis(self, version='v3', modelscope=False, **kwargs): @@ -58,19 +58,32 @@ class HM3VideoPipeline(HMPipeline): self.safety_checker.cpu() def insert_hm_modules(self, version='v3', dtype=torch.float16, modelscope=False): + self.version = version + if modelscope: from modelscope import snapshot_download - hm_reference_dir = snapshot_download('songkey/hm3_reference') - hm_control_dir = snapshot_download('songkey/hm3_control_mix') - hm_motion_dir = snapshot_download('songkey/hm3_motion') + if version == 'v3': + hm_reference_dir = snapshot_download('songkey/hm3_reference') + hm_control_dir = snapshot_download('songkey/hm3_control_mix') + hm_motion_dir = snapshot_download('songkey/hm3_motion') + else: + hm_reference_dir = snapshot_download('songkey/hm4_reference') + hm_control_dir = snapshot_download('songkey/hm_control_base') + hm_control_proj_dir = snapshot_download('songkey/hm4_control_proj') + hm_motion_dir = snapshot_download('songkey/hm4_motion') else: - hm_reference_dir = 'songkey/hm3_reference' - hm_control_dir = 'songkey/hm3_control_mix' - hm_motion_dir = 'songkey/hm3_motion' + if version == 'v3': + hm_reference_dir = 'songkey/hm3_reference' + hm_control_dir = 'songkey/hm3_control_mix' + hm_motion_dir = 'songkey/hm3_motion' + else: + hm_reference_dir = 'songkey/hm4_reference' + hm_control_dir = 'songkey/hm_control_base' + hm_control_proj_dir = 'songkey/hm4_control_proj' + hm_motion_dir = 'songkey/hm4_motion' hm_adapter = HM3ReferenceAdapter.from_pretrained(hm_reference_dir) motion_adapter = HM3MotionAdapter.from_pretrained(hm_motion_dir) - motion_adapter = motion_adapter.to(device='cpu', dtype=dtype).eval() if isinstance(self.unet, HM3DenoisingMotion): self.unet.insert_reference_adapter(hm_adapter) @@ -87,8 +100,16 @@ class HM3VideoPipeline(HMPipeline): if hasattr(self, "mp_control"): del self.mp_control + if hasattr(self, "mp_control_proj"): + del self.mp_control_proj - self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir) + if version == 'v3': + self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir) + else: + self.mp_control = HMControlNetBase.from_pretrained(hm_control_dir) + self.mp_control_proj = HM4SD15ControlProj.from_pretrained(hm_control_proj_dir) + + self.mp_control_proj.to(device='cpu', dtype=dtype).eval() self.mp_control.to(device='cpu', dtype=dtype).eval() self.vae.to(device='cpu', dtype=dtype).eval() @@ -456,6 +477,8 @@ class HM3VideoPipeline(HMPipeline): control_latents = [] control_latent1_neg, control_latent2_neg = None, None self.mp_control.to(device=device) + if self.version == 'v4': + self.mp_control_proj.to(device=device) with self.progress_bar(total=raw_video_len) as progress_bar: for idx in range(raw_video_len): progress_bar.set_description(f"GEN stage1") @@ -472,10 +495,14 @@ class HM3VideoPipeline(HMPipeline): control_latent1_neg = self.mp_control(condition=torch.ones_like(condition)*-1, drive_coeff=torch.zeros_like(drive_coeff), face_parts=torch.zeros_like(face_parts)) + if hasattr(self, 'mp_control_proj') and self.version == 'v4': + control_latent1_neg = self.mp_control_proj(control_latent1_neg) control_latent1 = self.mp_control(condition=condition, drive_coeff=drive_coeff, face_parts=face_parts) + if hasattr(self, 'mp_control_proj') and self.version == 'v4': + control_latent1 = self.mp_control_proj(control_latent1) if self.do_classifier_free_guidance: control_latent1 = cat_dicts([control_latent1_neg, control_latent1], dim=0) @@ -486,8 +513,12 @@ class HM3VideoPipeline(HMPipeline): if control_latent2_neg is None and self.do_classifier_free_guidance: control_latent2_neg = self.mp_control(condition=torch.ones_like(condition)*-1, emo_embedding=torch.zeros_like(pd_fpg)) + if hasattr(self, 'mp_control_proj') and self.version == 'v4': + control_latent2_neg = self.mp_control_proj(control_latent2_neg) control_latent2 = self.mp_control(condition=condition, emo_embedding=pd_fpg) + if hasattr(self, 'mp_control_proj') and self.version == 'v4': + control_latent2 = self.mp_control_proj(control_latent2) if self.do_classifier_free_guidance: control_latent2 = cat_dicts([control_latent2_neg, control_latent2], dim=0) @@ -495,6 +526,8 @@ class HM3VideoPipeline(HMPipeline): control_latents.append(dicts_to_device([control_latent_input], device='cpu')[0]) self.mp_control.cpu() + if self.version == 'v4': + self.mp_control_proj.cpu() more_params = dict( timesteps=timesteps, diff --git a/meme.py b/meme.py index fe7cf8b..407647f 100644 --- a/meme.py +++ b/meme.py @@ -124,7 +124,7 @@ class HMImagePipelineLoader: "checkpoint": (checkpoint_files, ), "lora": (lora_files, ), "vae": (vae_files, ), - "version": (['v1', 'v2', 'v3'], ), + "version": (['v1', 'v2', 'v3', 'v4'], ), "stylize": (['x1', 'x2'], ), "deployment": (['huggingface', 'modelscope'], ), "lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}), @@ -142,7 +142,7 @@ class HMImagePipelineLoader: sd1_5_dir = snapshot_download('songkey/stable-diffusion-v1-5') else: sd1_5_dir = "songkey/stable-diffusion-v1-5" - if version == 'v3': + if version == 'v3' or version == 'v4': pipeline = HM3ImagePipeline.from_pretrained(sd1_5_dir) else: pipeline = HMImagePipeline.from_pretrained(sd1_5_dir) @@ -166,7 +166,7 @@ class HMVideoPipelineLoader: "checkpoint": (checkpoint_files, ), "lora": (lora_files, ), "vae": (vae_files, ), - "version": (['v1', 'v2', 'v3'], ), + "version": (['v1', 'v2', 'v3', 'v4'], ), "stylize": (['x1', 'x2'], ), "deployment": (['huggingface', 'modelscope'], ), "lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}), @@ -187,7 +187,7 @@ class HMVideoPipelineLoader: else: sd1_5_dir = "songkey/stable-diffusion-v1-5" - if version == 'v3': + if version == 'v3' or version == 'v4': pipeline = HM3VideoPipeline.from_pretrained(sd1_5_dir) else: pipeline = HMVideoPipeline.from_pretrained(sd1_5_dir) diff --git a/pyproject.toml b/pyproject.toml index 4f6572a..07161fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_hellomeme" description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface" -version = "1.1.8" +version = "1.2.0" license = {file = "LICENSE"} [project.urls]