diff --git a/example_workflows/image_generation_ctrl1.json b/example_workflows/image_generation_ctrl1.json index f0597c4..21fa38a 100644 --- a/example_workflows/image_generation_ctrl1.json +++ b/example_workflows/image_generation_ctrl1.json @@ -1 +1 @@ -{"last_node_id":63,"last_link_id":129,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,127],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,126],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,125],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","shape":7,"link":124},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","shape":7,"link":null}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,859627586602455,"randomize",2,0]},{"id":61,"type":"GetDriveExpression","pos":[857,823],"size":[292.20001220703125,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":127},{"name":"images","type":"IMAGE","link":125},{"name":"landmarks","type":"FACELANDMARKS222","link":126}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[124],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[210,246],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v4","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[124,61,0,58,4,"DRIVE_EXPRESSION"],[125,57,0,61,1,"IMAGE"],[126,43,0,61,2,"FACELANDMARKS222"],[127,2,0,61,0,"FACE_TOOLKITS"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file +{"id":"351f5fbe-54ef-498e-8cd5-becb4a0c7d61","revision":0,"last_node_id":83,"last_link_id":195,"nodes":[{"id":61,"type":"GetFaceLandmarks","pos":[519.7196655273438,938.032958984375],"size":[170.62362670898438,46],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":129},{"localized_name":"images","name":"images","type":"IMAGE","link":187}],"outputs":[{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","links":[145,181]}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":2,"type":"HMFaceToolkitsLoader","pos":[45.41996383666992,769.9866333007812],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"gpu_id","name":"gpu_id","type":"INT","widget":{"name":"gpu_id"},"link":null},{"localized_name":"deployment","name":"deployment","type":"COMBO","widget":{"name":"deployment"},"link":null}],"outputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","slot_index":0,"links":[122,129,147,149,183,184]}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":60,"type":"GetFaceLandmarks","pos":[518.9132690429688,612.9461669921875],"size":[170.62362670898438,46],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":122},{"localized_name":"images","name":"images","type":"IMAGE","link":121}],"outputs":[{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","links":[143,185]}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":58,"type":"LoadImage","pos":[37.4466552734375,260.9000244140625],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","slot_index":0,"links":[121,151,186]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":79,"type":"LoadImage","pos":[52.432804107666016,1018.0931396484375],"size":[274.080078125,314.00006103515625],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[187,188,189]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["ysll.jpg","image"]},{"id":67,"type":"GetHeadPose","pos":[814.1528930664062,495.9793701171875],"size":[270,98],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":147},{"localized_name":"images","name":"images","type":"IMAGE","link":151},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":143},{"localized_name":"crop","name":"crop","type":"BOOLEAN","widget":{"name":"crop"},"link":null}],"outputs":[{"localized_name":"head_pose","name":"head_pose","type":"HEAD_POSE","links":[191]}],"properties":{"Node name for S&R":"GetHeadPose"},"widgets_values":[false]},{"id":77,"type":"GetExpression","pos":[821.1863403320312,695.4265747070312],"size":[170.62362670898438,66],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":184},{"localized_name":"images","name":"images","type":"IMAGE","link":186},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":185}],"outputs":[{"localized_name":"expression","name":"expression","type":"EXPRESSION","links":[192]}],"properties":{"Node name for S&R":"GetExpression"},"widgets_values":[]},{"id":69,"type":"GetHeadPose","pos":[819.799560546875,860.5927734375],"size":[270,98],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":149},{"localized_name":"images","name":"images","type":"IMAGE","link":188},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":145},{"localized_name":"crop","name":"crop","type":"BOOLEAN","widget":{"name":"crop"},"link":null}],"outputs":[{"localized_name":"head_pose","name":"head_pose","type":"HEAD_POSE","links":[193]}],"properties":{"Node name for S&R":"GetHeadPose"},"widgets_values":[true]},{"id":78,"type":"GetExpression","pos":[824.4126586914062,1061.6531982421875],"size":[170.62362670898438,66],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":183},{"localized_name":"images","name":"images","type":"IMAGE","link":189},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":181}],"outputs":[{"localized_name":"expression","name":"expression","type":"EXPRESSION","links":[194]}],"properties":{"Node name for S&R":"GetExpression"},"widgets_values":[]},{"id":80,"type":"HMPipelineImage","pos":[1275.33935546875,652.6725463867188],"size":[270,306],"flags":{},"order":10,"mode":0,"inputs":[{"localized_name":"hm_image_pipeline","name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":190},{"localized_name":"ref_head_pose","name":"ref_head_pose","type":"HEAD_POSE","link":191},{"localized_name":"ref_expression","name":"ref_expression","type":"EXPRESSION","link":192},{"localized_name":"drive_head_pose","name":"drive_head_pose","type":"HEAD_POSE","link":193},{"localized_name":"drive_expression","name":"drive_expression","type":"EXPRESSION","link":194},{"localized_name":"trans_ratio","name":"trans_ratio","type":"FLOAT","widget":{"name":"trans_ratio"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"negative_prompt","name":"negative_prompt","type":"STRING","widget":{"name":"negative_prompt"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"gpu_id","name":"gpu_id","type":"INT","widget":{"name":"gpu_id"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[195]},{"localized_name":"LATENT","name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,186205038040745,"randomize",2,0]},{"id":82,"type":"HMImagePipelineLoader","pos":[814.7327880859375,202.5532989501953],"size":[298.3472595214844,202],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"checkpoint","name":"checkpoint","shape":7,"type":"COMBO","widget":{"name":"checkpoint"},"link":null},{"localized_name":"lora","name":"lora","shape":7,"type":"COMBO","widget":{"name":"lora"},"link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"COMBO","widget":{"name":"vae"},"link":null},{"localized_name":"version","name":"version","shape":7,"type":"COMBO","widget":{"name":"version"},"link":null},{"localized_name":"stylize","name":"stylize","shape":7,"type":"COMBO","widget":{"name":"stylize"},"link":null},{"localized_name":"deployment","name":"deployment","shape":7,"type":"COMBO","widget":{"name":"deployment"},"link":null},{"localized_name":"lora_scale","name":"lora_scale","shape":7,"type":"FLOAT","widget":{"name":"lora_scale"},"link":null}],"outputs":[{"localized_name":"hm_image_pipeline","name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[190]}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","None","same as checkpoint","v4","x1","huggingface",1]},{"id":83,"type":"PreviewImage","pos":[1689.1593017578125,650.2534790039062],"size":[210,258],"flags":{},"order":11,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":195}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]}],"links":[[121,58,0,60,1,"IMAGE"],[122,2,0,60,0,"FACE_TOOLKITS"],[129,2,0,61,0,"FACE_TOOLKITS"],[143,60,0,67,2,"FACELANDMARKS222"],[145,61,0,69,2,"FACELANDMARKS222"],[147,2,0,67,0,"FACE_TOOLKITS"],[149,2,0,69,0,"FACE_TOOLKITS"],[151,58,0,67,1,"IMAGE"],[181,61,0,78,2,"FACELANDMARKS222"],[183,2,0,78,0,"FACE_TOOLKITS"],[184,2,0,77,0,"FACE_TOOLKITS"],[185,60,0,77,2,"FACELANDMARKS222"],[186,58,0,77,1,"IMAGE"],[187,79,0,61,1,"IMAGE"],[188,79,0,69,1,"IMAGE"],[189,79,0,78,1,"IMAGE"],[190,82,0,80,0,"HMIMAGEPIPELINE"],[191,67,0,80,1,"HEAD_POSE"],[192,77,0,80,2,"EXPRESSION"],[193,69,0,80,3,"HEAD_POSE"],[194,78,0,80,4,"EXPRESSION"],[195,80,0,83,0,"IMAGE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[607.8238861614929,3.14666035650405]},"frontendVersion":"1.19.2"},"version":0.4} \ No newline at end of file diff --git a/example_workflows/image_generation_ctrl2.json b/example_workflows/image_generation_ctrl2.json index da4fab7..105295b 100644 --- a/example_workflows/image_generation_ctrl2.json +++ b/example_workflows/image_generation_ctrl2.json @@ -1 +1 @@ -{"last_node_id":64,"last_link_id":133,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,132],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,130],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,131],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","shape":7,"link":null},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","shape":7,"link":133}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,404637369688717,"randomize",2,0]},{"id":64,"type":"GetDriveExpression2","pos":[850,827],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":132},{"name":"images","type":"IMAGE","link":131},{"name":"landmarks","type":"FACELANDMARKS222","link":130}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[133],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[290.7474365234375,283.26800537109375],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]MoreDetails","same as checkpoint","v4","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"],[130,43,0,64,2,"FACELANDMARKS222"],[131,57,0,64,1,"IMAGE"],[132,2,0,64,0,"FACE_TOOLKITS"],[133,64,0,58,5,"DRIVE_EXPRESSION2"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file +{"id":"4a7ae110-38ab-446e-97da-b826e950c065","revision":0,"last_node_id":85,"last_link_id":203,"nodes":[{"id":67,"type":"GetHeadPose","pos":[814.1528930664062,495.9793701171875],"size":[270,98],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":147},{"localized_name":"images","name":"images","type":"IMAGE","link":151},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":143},{"localized_name":"crop","name":"crop","type":"BOOLEAN","widget":{"name":"crop"},"link":null}],"outputs":[{"localized_name":"head_pose","name":"head_pose","type":"HEAD_POSE","links":[191]}],"properties":{"Node name for S&R":"GetHeadPose"},"widgets_values":[false]},{"id":69,"type":"GetHeadPose","pos":[819.799560546875,860.5927734375],"size":[270,98],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":149},{"localized_name":"images","name":"images","type":"IMAGE","link":188},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":145},{"localized_name":"crop","name":"crop","type":"BOOLEAN","widget":{"name":"crop"},"link":null}],"outputs":[{"localized_name":"head_pose","name":"head_pose","type":"HEAD_POSE","links":[193]}],"properties":{"Node name for S&R":"GetHeadPose"},"widgets_values":[true]},{"id":80,"type":"HMPipelineImage","pos":[1275.33935546875,652.6725463867188],"size":[270,306],"flags":{},"order":10,"mode":0,"inputs":[{"localized_name":"hm_image_pipeline","name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":190},{"localized_name":"ref_head_pose","name":"ref_head_pose","type":"HEAD_POSE","link":191},{"localized_name":"ref_expression","name":"ref_expression","type":"EXPRESSION","link":196},{"localized_name":"drive_head_pose","name":"drive_head_pose","type":"HEAD_POSE","link":193},{"localized_name":"drive_expression","name":"drive_expression","type":"EXPRESSION","link":197},{"localized_name":"trans_ratio","name":"trans_ratio","type":"FLOAT","widget":{"name":"trans_ratio"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"negative_prompt","name":"negative_prompt","type":"STRING","widget":{"name":"negative_prompt"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"gpu_id","name":"gpu_id","type":"INT","widget":{"name":"gpu_id"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[195]},{"localized_name":"LATENT","name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,1018802962613971,"randomize",2,0]},{"id":82,"type":"HMImagePipelineLoader","pos":[814.7327880859375,202.5532989501953],"size":[298.3472595214844,202],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"checkpoint","name":"checkpoint","shape":7,"type":"COMBO","widget":{"name":"checkpoint"},"link":null},{"localized_name":"lora","name":"lora","shape":7,"type":"COMBO","widget":{"name":"lora"},"link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"COMBO","widget":{"name":"vae"},"link":null},{"localized_name":"version","name":"version","shape":7,"type":"COMBO","widget":{"name":"version"},"link":null},{"localized_name":"stylize","name":"stylize","shape":7,"type":"COMBO","widget":{"name":"stylize"},"link":null},{"localized_name":"deployment","name":"deployment","shape":7,"type":"COMBO","widget":{"name":"deployment"},"link":null},{"localized_name":"lora_scale","name":"lora_scale","shape":7,"type":"FLOAT","widget":{"name":"lora_scale"},"link":null}],"outputs":[{"localized_name":"hm_image_pipeline","name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[190]}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","None","same as checkpoint","v4","x1","huggingface",1]},{"id":83,"type":"PreviewImage","pos":[1689.1593017578125,650.2534790039062],"size":[210,258],"flags":{},"order":11,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":195}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":84,"type":"GetExpression2","pos":[824.4129638671875,694.6199951171875],"size":[191.63729858398438,66],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":201},{"localized_name":"images","name":"images","type":"IMAGE","link":202},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":198}],"outputs":[{"localized_name":"expression","name":"expression","type":"EXPRESSION","links":[196]}],"properties":{"Node name for S&R":"GetExpression2"},"widgets_values":[]},{"id":85,"type":"GetExpression2","pos":[826.0261840820312,1060.0404052734375],"size":[191.63729858398438,66],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":200},{"localized_name":"images","name":"images","type":"IMAGE","link":203},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":199}],"outputs":[{"localized_name":"expression","name":"expression","type":"EXPRESSION","links":[197]}],"properties":{"Node name for S&R":"GetExpression2"},"widgets_values":[]},{"id":60,"type":"GetFaceLandmarks","pos":[518.9132690429688,612.9461669921875],"size":[170.62362670898438,46],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":122},{"localized_name":"images","name":"images","type":"IMAGE","link":121}],"outputs":[{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","links":[143,198]}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":61,"type":"GetFaceLandmarks","pos":[519.7196655273438,938.032958984375],"size":[170.62362670898438,46],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":129},{"localized_name":"images","name":"images","type":"IMAGE","link":187}],"outputs":[{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","links":[145,199]}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":2,"type":"HMFaceToolkitsLoader","pos":[45.41996383666992,769.9866333007812],"size":[315,82],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"gpu_id","name":"gpu_id","type":"INT","widget":{"name":"gpu_id"},"link":null},{"localized_name":"deployment","name":"deployment","type":"COMBO","widget":{"name":"deployment"},"link":null}],"outputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","slot_index":0,"links":[122,129,147,149,200,201]}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":58,"type":"LoadImage","pos":[37.4466552734375,260.9000244140625],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","slot_index":0,"links":[121,151,202]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":79,"type":"LoadImage","pos":[52.432804107666016,1018.0931396484375],"size":[274.080078125,314.00006103515625],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[187,188,203]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["ysll.jpg","image"]}],"links":[[121,58,0,60,1,"IMAGE"],[122,2,0,60,0,"FACE_TOOLKITS"],[129,2,0,61,0,"FACE_TOOLKITS"],[143,60,0,67,2,"FACELANDMARKS222"],[145,61,0,69,2,"FACELANDMARKS222"],[147,2,0,67,0,"FACE_TOOLKITS"],[149,2,0,69,0,"FACE_TOOLKITS"],[151,58,0,67,1,"IMAGE"],[187,79,0,61,1,"IMAGE"],[188,79,0,69,1,"IMAGE"],[190,82,0,80,0,"HMIMAGEPIPELINE"],[191,67,0,80,1,"HEAD_POSE"],[193,69,0,80,3,"HEAD_POSE"],[195,80,0,83,0,"IMAGE"],[196,84,0,80,2,"EXPRESSION"],[197,85,0,80,4,"EXPRESSION"],[198,60,0,84,2,"FACELANDMARKS222"],[199,61,0,85,2,"FACELANDMARKS222"],[200,2,0,85,0,"FACE_TOOLKITS"],[201,2,0,84,0,"FACE_TOOLKITS"],[202,58,0,84,1,"IMAGE"],[203,79,0,85,1,"IMAGE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[459.39712102477415,-14.59995707513657]},"frontendVersion":"1.19.2"},"version":0.4} \ No newline at end of file diff --git a/example_workflows/video_generation_ctrl1.json b/example_workflows/video_generation_ctrl1.json index 299a1f6..95a3d99 100644 --- a/example_workflows/video_generation_ctrl1.json +++ b/example_workflows/video_generation_ctrl1.json @@ -1,682 +1 @@ -{ - "last_node_id": 60, - "last_link_id": 122, - "nodes": [ - { - "id": 2, - "type": "HMFaceToolkitsLoader", - "pos": [ - 43, - 699 - ], - "size": [ - 315, - 82 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "links": [ - 57, - 62, - 65, - 97, - 119 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMFaceToolkitsLoader" - }, - "widgets_values": [ - 0, - "huggingface" - ] - }, - { - "id": 39, - "type": "CropPortrait", - "pos": [ - 516, - 537 - ], - "size": [ - 241.79998779296875, - 46 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 116 - }, - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 57 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 99 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CropPortrait" - }, - "widgets_values": [] - }, - { - "id": 43, - "type": "GetFaceLandmarks", - "pos": [ - 465, - 771 - ], - "size": [ - 292.20001220703125, - 46 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 62 - }, - { - "name": "images", - "type": "IMAGE", - "link": 82 - } - ], - "outputs": [ - { - "name": "landmarks", - "type": "FACELANDMARKS222", - "links": [ - 64, - 120 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetFaceLandmarks" - }, - "widgets_values": [] - }, - { - "id": 44, - "type": "GetDrivePose", - "pos": [ - 842, - 677 - ], - "size": [ - 304.79998779296875, - 66 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 65 - }, - { - "name": "images", - "type": "IMAGE", - "link": 83 - }, - { - "name": "landmarks", - "type": "FACELANDMARKS222", - "link": 64 - } - ], - "outputs": [ - { - "name": "drive_pose", - "type": "DRIVE_POSE", - "links": [ - 100 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetDrivePose" - }, - "widgets_values": [] - }, - { - "id": 53, - "type": "HMPipelineVideo", - "pos": [ - 1258, - 562 - ], - "size": [ - 315, - 350 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "hm_video_pipeline", - "type": "HMVIDEOPIPELINE", - "link": 115 - }, - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 97 - }, - { - "name": "ref_image", - "type": "IMAGE", - "link": 99 - }, - { - "name": "drive_pose", - "type": "DRIVE_POSE", - "link": 100 - }, - { - "name": "drive_exp", - "type": "DRIVE_EXPRESSION", - "shape": 7, - "link": 122 - }, - { - "name": "drive_exp2", - "type": "DRIVE_EXPRESSION2", - "shape": 7, - "link": null - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 117 - ], - "slot_index": 0 - }, - { - "name": "LATENT", - "type": "LATENT", - "links": null - } - ], - "properties": { - "Node name for S&R": "HMPipelineVideo" - }, - "widgets_values": [ - 0, - 4, - "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", - "", - 25, - 990809923057900, - "randomize", - 2, - 0 - ] - }, - { - "id": 60, - "type": "GetDriveExpression", - "pos": [ - 845, - 813 - ], - "size": [ - 292.20001220703125, - 66 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 119 - }, - { - "name": "images", - "type": "IMAGE", - "link": 121 - }, - { - "name": "landmarks", - "type": "FACELANDMARKS222", - "link": 120 - } - ], - "outputs": [ - { - "name": "drive_exp", - "type": "DRIVE_EXPRESSION", - "links": [ - 122 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetDriveExpression" - }, - "widgets_values": [] - }, - { - "id": 58, - "type": "LoadImage", - "pos": [ - 56, - 273 - ], - "size": [ - 315, - 314 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 116 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "toon.png", - "image" - ] - }, - { - "id": 59, - "type": "VHS_VideoCombine", - "pos": [ - 1619, - 800 - ], - "size": [ - 280.4215087890625, - 584.4215087890625 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 117 - }, - { - "name": "audio", - "type": "AUDIO", - "shape": 7, - "link": 118 - }, - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "shape": 7, - "link": null - }, - { - "name": "vae", - "type": "VAE", - "shape": 7, - "link": null - } - ], - "outputs": [ - { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": null - } - ], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": { - "frame_rate": 15, - "loop_count": 0, - "filename_prefix": "AnimateDiff", - "format": "video/h264-mp4", - "pix_fmt": "yuv420p", - "crf": 18, - "save_metadata": true, - "pingpong": false, - "save_output": true, - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "AnimateDiff_00409-audio.mp4", - "subfolder": "", - "type": "output", - "format": "video/h264-mp4", - "frame_rate": 15 - }, - "muted": false - } - } - }, - { - "id": 48, - "type": "VHS_LoadVideo", - "pos": [ - 106, - 867 - ], - "size": [ - 247.455078125, - 304 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "shape": 7, - "link": null - }, - { - "name": "vae", - "type": "VAE", - "shape": 7, - "link": null - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 82, - 83, - 121 - ], - "slot_index": 0 - }, - { - "name": "frame_count", - "type": "INT", - "links": null - }, - { - "name": "audio", - "type": "AUDIO", - "links": [ - 118 - ], - "slot_index": 2 - }, - { - "name": "video_info", - "type": "VHS_VIDEOINFO", - "links": null - } - ], - "properties": { - "Node name for S&R": "VHS_LoadVideo" - }, - "widgets_values": { - "video": "tiktok.mp4", - "force_rate": 15, - "force_size": "Disabled", - "custom_width": 512, - "custom_height": 512, - "frame_load_cap": 0, - "skip_first_frames": 0, - "select_every_nth": 1, - "choose video to upload": "image", - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "force_rate": 15, - "frame_load_cap": 0, - "skip_first_frames": 0, - "select_every_nth": 1, - "filename": "tiktok.mp4", - "type": "input", - "format": "video/mp4" - }, - "muted": false - } - } - }, - { - "id": 57, - "type": "HMVideoPipelineLoader", - "pos": [ - 813, - 343 - ], - "size": [ - 352.79998779296875, - 202 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "hm_video_pipeline", - "type": "HMVIDEOPIPELINE", - "links": [ - 115 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMVideoPipelineLoader" - }, - "widgets_values": [ - "[preset]DisneyPixarCartoonB", - "[preset]MoreDetails", - "same as checkpoint", - "v4", - "x1", - "huggingface", - 1 - ] - } - ], - "links": [ - [ - 57, - 2, - 0, - 39, - 1, - "FACE_TOOLKITS" - ], - [ - 62, - 2, - 0, - 43, - 0, - "FACE_TOOLKITS" - ], - [ - 64, - 43, - 0, - 44, - 2, - "FACELANDMARKS222" - ], - [ - 65, - 2, - 0, - 44, - 0, - "FACE_TOOLKITS" - ], - [ - 82, - 48, - 0, - 43, - 1, - "IMAGE" - ], - [ - 83, - 48, - 0, - 44, - 1, - "IMAGE" - ], - [ - 97, - 2, - 0, - 53, - 1, - "FACE_TOOLKITS" - ], - [ - 99, - 39, - 0, - 53, - 2, - "IMAGE" - ], - [ - 100, - 44, - 0, - 53, - 3, - "DRIVE_POSE" - ], - [ - 115, - 57, - 0, - 53, - 0, - "HMVIDEOPIPELINE" - ], - [ - 116, - 58, - 0, - 39, - 0, - "IMAGE" - ], - [ - 117, - 53, - 0, - 59, - 0, - "IMAGE" - ], - [ - 118, - 48, - 2, - 59, - 1, - "AUDIO" - ], - [ - 119, - 2, - 0, - 60, - 0, - "FACE_TOOLKITS" - ], - [ - 120, - 43, - 0, - 60, - 2, - "FACELANDMARKS222" - ], - [ - 121, - 48, - 0, - 60, - 1, - "IMAGE" - ], - [ - 122, - 60, - 0, - 53, - 4, - "DRIVE_EXPRESSION" - ] - ], - "groups": [], - "config": {}, - "extra": { - "ds": { - "scale": 0.7513148009015779, - "offset": [ - 634.1362099561715, - -70.9744212566187 - ] - } - }, - "version": 0.4 -} \ No newline at end of file +{"id":"c90d8660-52ae-4012-9627-95509cdfad22","revision":0,"last_node_id":78,"last_link_id":186,"nodes":[{"id":67,"type":"GetHeadPose","pos":[814.1528930664062,495.9793701171875],"size":[270,98],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":147},{"localized_name":"images","name":"images","type":"IMAGE","link":151},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":143},{"localized_name":"crop","name":"crop","type":"BOOLEAN","widget":{"name":"crop"},"link":null}],"outputs":[{"localized_name":"head_pose","name":"head_pose","type":"HEAD_POSE","links":[166]}],"properties":{"Node name for S&R":"GetHeadPose"},"widgets_values":[false]},{"id":59,"type":"VHS_VideoCombine","pos":[1658.52490234375,685.6950073242188],"size":[329.22479248046875,310],"flags":{},"order":11,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":170},{"localized_name":"audio","name":"audio","shape":7,"type":"AUDIO","link":118},{"localized_name":"meta_batch","name":"meta_batch","shape":7,"type":"VHS_BatchManager","link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"VAE","link":null},{"localized_name":"frame_rate","name":"frame_rate","type":"FLOAT","widget":{"name":"frame_rate"},"link":null},{"localized_name":"loop_count","name":"loop_count","type":"INT","widget":{"name":"loop_count"},"link":null},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"pingpong","name":"pingpong","type":"BOOLEAN","widget":{"name":"pingpong"},"link":null},{"localized_name":"save_output","name":"save_output","type":"BOOLEAN","widget":{"name":"save_output"},"link":null}],"outputs":[{"localized_name":"Filenames","name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00611-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":74,"type":"HMPipelineVideo","pos":[1296.2967529296875,659.4104614257812],"size":[270,330],"flags":{},"order":10,"mode":0,"inputs":[{"localized_name":"hm_video_pipeline","name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":165},{"localized_name":"ref_head_pose","name":"ref_head_pose","type":"HEAD_POSE","link":166},{"localized_name":"ref_expression","name":"ref_expression","type":"EXPRESSION","link":179},{"localized_name":"drive_head_pose","name":"drive_head_pose","type":"HEAD_POSE","link":168},{"localized_name":"drive_expression","name":"drive_expression","type":"EXPRESSION","link":180},{"localized_name":"trans_ratio","name":"trans_ratio","type":"FLOAT","widget":{"name":"trans_ratio"},"link":null},{"localized_name":"patch_overlap","name":"patch_overlap","type":"INT","widget":{"name":"patch_overlap"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"negative_prompt","name":"negative_prompt","type":"STRING","widget":{"name":"negative_prompt"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"gpu_id","name":"gpu_id","type":"INT","widget":{"name":"gpu_id"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[170]},{"localized_name":"LATENT","name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,525321077489364,"randomize",3.5,0]},{"id":57,"type":"HMVideoPipelineLoader","pos":[798.4796752929688,193.7666015625],"size":[352.79998779296875,202],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"checkpoint","name":"checkpoint","shape":7,"type":"COMBO","widget":{"name":"checkpoint"},"link":null},{"localized_name":"lora","name":"lora","shape":7,"type":"COMBO","widget":{"name":"lora"},"link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"COMBO","widget":{"name":"vae"},"link":null},{"localized_name":"version","name":"version","shape":7,"type":"COMBO","widget":{"name":"version"},"link":null},{"localized_name":"stylize","name":"stylize","shape":7,"type":"COMBO","widget":{"name":"stylize"},"link":null},{"localized_name":"deployment","name":"deployment","shape":7,"type":"COMBO","widget":{"name":"deployment"},"link":null},{"localized_name":"lora_scale","name":"lora_scale","shape":7,"type":"FLOAT","widget":{"name":"lora_scale"},"link":null}],"outputs":[{"localized_name":"hm_video_pipeline","name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","slot_index":0,"links":[165]}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","None","same as checkpoint","v4","x1","huggingface",1]},{"id":69,"type":"GetHeadPose","pos":[819.799560546875,860.5927734375],"size":[270,98],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":149},{"localized_name":"images","name":"images","type":"IMAGE","link":153},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":145},{"localized_name":"crop","name":"crop","type":"BOOLEAN","widget":{"name":"crop"},"link":null}],"outputs":[{"localized_name":"head_pose","name":"head_pose","type":"HEAD_POSE","links":[168]}],"properties":{"Node name for S&R":"GetHeadPose"},"widgets_values":[true]},{"id":77,"type":"GetExpression","pos":[821.1863403320312,695.4265747070312],"size":[170.62362670898438,66],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":184},{"localized_name":"images","name":"images","type":"IMAGE","link":186},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":185}],"outputs":[{"localized_name":"expression","name":"expression","type":"EXPRESSION","links":[179]}],"properties":{"Node name for S&R":"GetExpression"},"widgets_values":[]},{"id":61,"type":"GetFaceLandmarks","pos":[519.7196655273438,938.032958984375],"size":[170.62362670898438,46],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":129},{"localized_name":"images","name":"images","type":"IMAGE","link":131}],"outputs":[{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","links":[145,181]}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":48,"type":"VHS_LoadVideo","pos":[74.54000091552734,996.8733520507812],"size":[247.455078125,262],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"meta_batch","name":"meta_batch","shape":7,"type":"VHS_BatchManager","link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"VAE","link":null},{"localized_name":"video","name":"video","type":"COMBO","widget":{"name":"video"},"link":null},{"localized_name":"force_rate","name":"force_rate","type":"INT","widget":{"name":"force_rate"},"link":null},{"localized_name":"force_size","name":"force_size","type":"COMBO","widget":{"name":"force_size"},"link":null},{"localized_name":"custom_width","name":"custom_width","type":"INT","widget":{"name":"custom_width"},"link":null},{"localized_name":"custom_height","name":"custom_height","type":"INT","widget":{"name":"custom_height"},"link":null},{"localized_name":"frame_load_cap","name":"frame_load_cap","type":"INT","widget":{"name":"frame_load_cap"},"link":null},{"localized_name":"skip_first_frames","name":"skip_first_frames","type":"INT","widget":{"name":"skip_first_frames"},"link":null},{"localized_name":"select_every_nth","name":"select_every_nth","type":"INT","widget":{"name":"select_every_nth"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","slot_index":0,"links":[131,153,182]},{"localized_name":"frame_count","name":"frame_count","type":"INT","links":null},{"localized_name":"audio","name":"audio","type":"AUDIO","slot_index":2,"links":[118]},{"localized_name":"video_info","name":"video_info","type":"VHS_VIDEOINFO","links":null}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"tiktok.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"tiktok.mp4","type":"input","format":"video/mp4"},"muted":false}}},{"id":2,"type":"HMFaceToolkitsLoader","pos":[45.41996383666992,769.9866333007812],"size":[315,82],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"gpu_id","name":"gpu_id","type":"INT","widget":{"name":"gpu_id"},"link":null},{"localized_name":"deployment","name":"deployment","type":"COMBO","widget":{"name":"deployment"},"link":null}],"outputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","slot_index":0,"links":[122,129,147,149,183,184]}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":60,"type":"GetFaceLandmarks","pos":[518.9132690429688,612.9461669921875],"size":[170.62362670898438,46],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":122},{"localized_name":"images","name":"images","type":"IMAGE","link":121}],"outputs":[{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","links":[143,185]}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":58,"type":"LoadImage","pos":[37.4466552734375,260.9000244140625],"size":[315,314],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","slot_index":0,"links":[121,151,186]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":78,"type":"GetExpression","pos":[824.4126586914062,1061.6531982421875],"size":[170.62362670898438,66],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":183},{"localized_name":"images","name":"images","type":"IMAGE","link":182},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":181}],"outputs":[{"localized_name":"expression","name":"expression","type":"EXPRESSION","links":[180]}],"properties":{"Node name for S&R":"GetExpression"},"widgets_values":[]}],"links":[[118,48,2,59,1,"AUDIO"],[121,58,0,60,1,"IMAGE"],[122,2,0,60,0,"FACE_TOOLKITS"],[129,2,0,61,0,"FACE_TOOLKITS"],[131,48,0,61,1,"IMAGE"],[143,60,0,67,2,"FACELANDMARKS222"],[145,61,0,69,2,"FACELANDMARKS222"],[147,2,0,67,0,"FACE_TOOLKITS"],[149,2,0,69,0,"FACE_TOOLKITS"],[151,58,0,67,1,"IMAGE"],[153,48,0,69,1,"IMAGE"],[165,57,0,74,0,"HMVIDEOPIPELINE"],[166,67,0,74,1,"HEAD_POSE"],[168,69,0,74,3,"HEAD_POSE"],[170,74,0,59,0,"IMAGE"],[179,77,0,74,2,"EXPRESSION"],[180,78,0,74,4,"EXPRESSION"],[181,61,0,78,2,"FACELANDMARKS222"],[182,48,0,78,1,"IMAGE"],[183,2,0,78,0,"FACE_TOOLKITS"],[184,2,0,77,0,"FACE_TOOLKITS"],[185,60,0,77,2,"FACELANDMARKS222"],[186,58,0,77,1,"IMAGE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[607.8238861614929,3.14666035650405]},"frontendVersion":"1.19.2"},"version":0.4} \ No newline at end of file diff --git a/example_workflows/video_generation_ctrl2.json b/example_workflows/video_generation_ctrl2.json index b1b66c2..b097f09 100644 --- a/example_workflows/video_generation_ctrl2.json +++ b/example_workflows/video_generation_ctrl2.json @@ -1 +1 @@ -{"last_node_id":59,"last_link_id":118,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,114],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"image","localized_name":"image","type":"IMAGE","link":116},{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","localized_name":"图像","type":"IMAGE","links":[99],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","localized_name":"images","type":"IMAGE","link":82}],"outputs":[{"name":"landmarks","localized_name":"landmarks","type":"FACELANDMARKS222","links":[64,113],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","localized_name":"images","type":"IMAGE","link":83},{"name":"landmarks","localized_name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","localized_name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_video_pipeline","localized_name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115},{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":97},{"name":"ref_image","localized_name":"ref_image","type":"IMAGE","link":99},{"name":"drive_pose","localized_name":"drive_pose","type":"DRIVE_POSE","link":100},{"name":"drive_exp","localized_name":"drive_exp","type":"DRIVE_EXPRESSION","shape":7,"link":null},{"name":"drive_exp2","localized_name":"drive_exp2","type":"DRIVE_EXPRESSION2","shape":7,"link":111}],"outputs":[{"name":"IMAGE","localized_name":"图像","type":"IMAGE","links":[117],"slot_index":0},{"name":"LATENT","localized_name":"Latent","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,293728504444278,"randomize",2,0]},{"id":56,"type":"GetDriveExpression2","pos":[844,816],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","localized_name":"face_toolkits","type":"FACE_TOOLKITS","link":114},{"name":"images","localized_name":"images","type":"IMAGE","link":112},{"name":"landmarks","localized_name":"landmarks","type":"FACELANDMARKS222","link":113}],"outputs":[{"name":"drive_exp2","localized_name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[111],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":59,"type":"VHS_VideoCombine","pos":[1668.69091796875,739.6614379882812],"size":[329.22479248046875,633.2247924804688],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","localized_name":"images","type":"IMAGE","link":117},{"name":"audio","localized_name":"audio","type":"AUDIO","shape":7,"link":118},{"name":"meta_batch","localized_name":"meta_batch","type":"VHS_BatchManager","shape":7,"link":null},{"name":"vae","localized_name":"vae","type":"VAE","shape":7,"link":null}],"outputs":[{"name":"Filenames","localized_name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00408-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","localized_name":"图像","type":"IMAGE","links":[116],"slot_index":0},{"name":"MASK","localized_name":"遮罩","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":48,"type":"VHS_LoadVideo","pos":[106,867],"size":[247.455078125,304],"flags":{},"order":2,"mode":0,"inputs":[{"name":"meta_batch","localized_name":"meta_batch","type":"VHS_BatchManager","shape":7,"link":null},{"name":"vae","localized_name":"vae","type":"VAE","shape":7,"link":null}],"outputs":[{"name":"IMAGE","localized_name":"图像","type":"IMAGE","links":[82,83,112],"slot_index":0},{"name":"frame_count","localized_name":"frame_count","type":"INT","links":null},{"name":"audio","localized_name":"audio","type":"AUDIO","links":[118],"slot_index":2},{"name":"video_info","localized_name":"video_info","type":"VHS_VIDEOINFO","links":null}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"tiktok.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"tiktok.mp4","type":"input","format":"video/mp4"},"muted":false}}},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","localized_name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]MoreDetails","same as checkpoint","v4","x1","huggingface",1]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[82,48,0,43,1,"IMAGE"],[83,48,0,44,1,"IMAGE"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[111,56,0,53,5,"DRIVE_EXPRESSION2"],[112,48,0,56,1,"IMAGE"],[113,43,0,56,2,"FACELANDMARKS222"],[114,2,0,56,0,"FACE_TOOLKITS"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[117,53,0,59,0,"IMAGE"],[118,48,2,59,1,"AUDIO"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file +{"id":"4c8b7397-a69c-46cc-b553-00e600557365","revision":0,"last_node_id":76,"last_link_id":178,"nodes":[{"id":67,"type":"GetHeadPose","pos":[814.1528930664062,495.9793701171875],"size":[270,98],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":147},{"localized_name":"images","name":"images","type":"IMAGE","link":151},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":143},{"localized_name":"crop","name":"crop","type":"BOOLEAN","widget":{"name":"crop"},"link":null}],"outputs":[{"localized_name":"head_pose","name":"head_pose","type":"HEAD_POSE","links":[166]}],"properties":{"Node name for S&R":"GetHeadPose"},"widgets_values":[false]},{"id":75,"type":"GetExpression2","pos":[814.1359252929688,689.6795654296875],"size":[191.63729858398438,66],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":173},{"localized_name":"images","name":"images","type":"IMAGE","link":175},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":178}],"outputs":[{"localized_name":"expression","name":"expression","type":"EXPRESSION","links":[171]}],"properties":{"Node name for S&R":"GetExpression2"},"widgets_values":[]},{"id":76,"type":"GetExpression2","pos":[833.6573486328125,1042.0396728515625],"size":[191.63729858398438,66],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":174},{"localized_name":"images","name":"images","type":"IMAGE","link":176},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":177}],"outputs":[{"localized_name":"expression","name":"expression","type":"EXPRESSION","links":[172]}],"properties":{"Node name for S&R":"GetExpression2"},"widgets_values":[]},{"id":2,"type":"HMFaceToolkitsLoader","pos":[45.41996383666992,769.9866333007812],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"gpu_id","name":"gpu_id","type":"INT","widget":{"name":"gpu_id"},"link":null},{"localized_name":"deployment","name":"deployment","type":"COMBO","widget":{"name":"deployment"},"link":null}],"outputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","slot_index":0,"links":[122,129,147,149,173,174]}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":61,"type":"GetFaceLandmarks","pos":[519.7196655273438,938.032958984375],"size":[170.62362670898438,46],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":129},{"localized_name":"images","name":"images","type":"IMAGE","link":131}],"outputs":[{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","links":[145,177]}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":60,"type":"GetFaceLandmarks","pos":[518.9132690429688,612.9461669921875],"size":[170.62362670898438,46],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":122},{"localized_name":"images","name":"images","type":"IMAGE","link":121}],"outputs":[{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","links":[143,178]}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":59,"type":"VHS_VideoCombine","pos":[1658.52490234375,685.6950073242188],"size":[329.22479248046875,633.2247924804688],"flags":{},"order":11,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":170},{"localized_name":"audio","name":"audio","shape":7,"type":"AUDIO","link":118},{"localized_name":"meta_batch","name":"meta_batch","shape":7,"type":"VHS_BatchManager","link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"VAE","link":null},{"localized_name":"frame_rate","name":"frame_rate","type":"FLOAT","widget":{"name":"frame_rate"},"link":null},{"localized_name":"loop_count","name":"loop_count","type":"INT","widget":{"name":"loop_count"},"link":null},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"pingpong","name":"pingpong","type":"BOOLEAN","widget":{"name":"pingpong"},"link":null},{"localized_name":"save_output","name":"save_output","type":"BOOLEAN","widget":{"name":"save_output"},"link":null}],"outputs":[{"localized_name":"Filenames","name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00611-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":74,"type":"HMPipelineVideo","pos":[1296.2967529296875,659.4104614257812],"size":[270,330],"flags":{},"order":10,"mode":0,"inputs":[{"localized_name":"hm_video_pipeline","name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":165},{"localized_name":"ref_head_pose","name":"ref_head_pose","type":"HEAD_POSE","link":166},{"localized_name":"ref_expression","name":"ref_expression","type":"EXPRESSION","link":171},{"localized_name":"drive_head_pose","name":"drive_head_pose","type":"HEAD_POSE","link":168},{"localized_name":"drive_expression","name":"drive_expression","type":"EXPRESSION","link":172},{"localized_name":"trans_ratio","name":"trans_ratio","type":"FLOAT","widget":{"name":"trans_ratio"},"link":null},{"localized_name":"patch_overlap","name":"patch_overlap","type":"INT","widget":{"name":"patch_overlap"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"negative_prompt","name":"negative_prompt","type":"STRING","widget":{"name":"negative_prompt"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"gpu_id","name":"gpu_id","type":"INT","widget":{"name":"gpu_id"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[170]},{"localized_name":"LATENT","name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,1055114486951082,"randomize",3.5,0]},{"id":57,"type":"HMVideoPipelineLoader","pos":[798.4796752929688,193.7666015625],"size":[352.79998779296875,202],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"checkpoint","name":"checkpoint","shape":7,"type":"COMBO","widget":{"name":"checkpoint"},"link":null},{"localized_name":"lora","name":"lora","shape":7,"type":"COMBO","widget":{"name":"lora"},"link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"COMBO","widget":{"name":"vae"},"link":null},{"localized_name":"version","name":"version","shape":7,"type":"COMBO","widget":{"name":"version"},"link":null},{"localized_name":"stylize","name":"stylize","shape":7,"type":"COMBO","widget":{"name":"stylize"},"link":null},{"localized_name":"deployment","name":"deployment","shape":7,"type":"COMBO","widget":{"name":"deployment"},"link":null},{"localized_name":"lora_scale","name":"lora_scale","shape":7,"type":"FLOAT","widget":{"name":"lora_scale"},"link":null}],"outputs":[{"localized_name":"hm_video_pipeline","name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","slot_index":0,"links":[165]}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","None","same as checkpoint","v4","x1","huggingface",1]},{"id":58,"type":"LoadImage","pos":[37.4466552734375,260.9000244140625],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","slot_index":0,"links":[121,151,175]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":69,"type":"GetHeadPose","pos":[819.799560546875,860.5927734375],"size":[270,98],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"face_toolkits","name":"face_toolkits","type":"FACE_TOOLKITS","link":149},{"localized_name":"images","name":"images","type":"IMAGE","link":153},{"localized_name":"landmarks","name":"landmarks","type":"FACELANDMARKS222","link":145},{"localized_name":"crop","name":"crop","type":"BOOLEAN","widget":{"name":"crop"},"link":null}],"outputs":[{"localized_name":"head_pose","name":"head_pose","type":"HEAD_POSE","links":[168]}],"properties":{"Node name for S&R":"GetHeadPose"},"widgets_values":[true]},{"id":48,"type":"VHS_LoadVideo","pos":[74.54000091552734,996.8733520507812],"size":[247.455078125,680.3645629882812],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"meta_batch","name":"meta_batch","shape":7,"type":"VHS_BatchManager","link":null},{"localized_name":"vae","name":"vae","shape":7,"type":"VAE","link":null},{"localized_name":"video","name":"video","type":"COMBO","widget":{"name":"video"},"link":null},{"localized_name":"force_rate","name":"force_rate","type":"INT","widget":{"name":"force_rate"},"link":null},{"localized_name":"force_size","name":"force_size","type":"COMBO","widget":{"name":"force_size"},"link":null},{"localized_name":"custom_width","name":"custom_width","type":"INT","widget":{"name":"custom_width"},"link":null},{"localized_name":"custom_height","name":"custom_height","type":"INT","widget":{"name":"custom_height"},"link":null},{"localized_name":"frame_load_cap","name":"frame_load_cap","type":"INT","widget":{"name":"frame_load_cap"},"link":null},{"localized_name":"skip_first_frames","name":"skip_first_frames","type":"INT","widget":{"name":"skip_first_frames"},"link":null},{"localized_name":"select_every_nth","name":"select_every_nth","type":"INT","widget":{"name":"select_every_nth"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","slot_index":0,"links":[131,153,176]},{"localized_name":"frame_count","name":"frame_count","type":"INT","links":null},{"localized_name":"audio","name":"audio","type":"AUDIO","slot_index":2,"links":[118]},{"localized_name":"video_info","name":"video_info","type":"VHS_VIDEOINFO","links":null}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"tiktok.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"tiktok.mp4","type":"input","format":"video/mp4"},"muted":false}}}],"links":[[118,48,2,59,1,"AUDIO"],[121,58,0,60,1,"IMAGE"],[122,2,0,60,0,"FACE_TOOLKITS"],[129,2,0,61,0,"FACE_TOOLKITS"],[131,48,0,61,1,"IMAGE"],[143,60,0,67,2,"FACELANDMARKS222"],[145,61,0,69,2,"FACELANDMARKS222"],[147,2,0,67,0,"FACE_TOOLKITS"],[149,2,0,69,0,"FACE_TOOLKITS"],[151,58,0,67,1,"IMAGE"],[153,48,0,69,1,"IMAGE"],[165,57,0,74,0,"HMVIDEOPIPELINE"],[166,67,0,74,1,"HEAD_POSE"],[168,69,0,74,3,"HEAD_POSE"],[170,74,0,59,0,"IMAGE"],[171,75,0,74,2,"EXPRESSION"],[172,76,0,74,4,"EXPRESSION"],[173,2,0,75,0,"FACE_TOOLKITS"],[174,2,0,76,0,"FACE_TOOLKITS"],[175,58,0,75,1,"IMAGE"],[176,48,0,76,1,"IMAGE"],[177,61,0,76,2,"FACELANDMARKS222"],[178,60,0,75,2,"FACELANDMARKS222"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[605.403886161493,-0.08003092767563647]},"frontendVersion":"1.19.2"},"version":0.4} \ No newline at end of file diff --git a/hellomeme/utils.py b/hellomeme/utils.py index 02b99ea..f1816ac 100644 --- a/hellomeme/utils.py +++ b/hellomeme/utils.py @@ -213,13 +213,11 @@ face_region = [33, 74, 96, 133, 149] face_temp = np.array([[0.50951539, 0.40128625], [0.29259476, 0.40203411], [0.72196212, 0.40857479], [0.63420714, 0.71678643], [0.38437035, 0.71723224]]) -def get_drive_pose(toolkits, frames, landmarks, save_size=512, align=True): +def get_drive_pose(toolkits, frames, landmarks, save_size=512, crop=True): if isinstance(landmarks, list): landmarks = np.stack(landmarks, axis=0) - if align: - new_frames, new_landmarks = crop_and_resize(np.stack(frames, axis=0), landmarks, - save_size=save_size, crop=False) - else: - new_frames, new_landmarks = frames, landmarks + + new_frames, new_landmarks = crop_and_resize(np.stack(frames, axis=0), landmarks, + save_size=save_size, crop=crop) rot_list = [] trans_list = [] @@ -231,7 +229,7 @@ def get_drive_pose(toolkits, frames, landmarks, save_size=512, align=True): pbar.set_description('HEAD POSE') pbar.update() - return rot_list, trans_list + return new_frames, new_landmarks, rot_list, trans_list def get_drive_expression(toolkits, images, landmarks): diff --git a/meme.py b/meme.py index 7d7f6d4..5f57a9d 100644 --- a/meme.py +++ b/meme.py @@ -47,7 +47,6 @@ from .hellomeme.utils import (get_drive_expression, get_drive_expression_pd_fgc, gen_control_heatmaps, get_drive_pose, - crop_and_resize, det_landmarks, get_torch_device, append_pipline_weights, @@ -147,11 +146,11 @@ class HMImagePipelineLoader: else: pipeline = HMImagePipeline.from_pretrained(sd1_5_dir) pipeline.to(dtype=dtype) - pipeline.caryomitosis(version=version, modelscope=deployment=='modelscope') + pipeline.caryomitosis(version=version, modelscope=deployment == 'modelscope') format_model_path(pipeline, MODEL_CONFIG, checkpoint, vae, lora, stylize, lora_scale, deployment) - pipeline.insert_hm_modules(version=version, dtype=dtype, modelscope=deployment=='modelscope') + pipeline.insert_hm_modules(version=version, dtype=dtype, modelscope=deployment == 'modelscope') return (pipeline, ) @@ -222,42 +221,6 @@ class HMFaceToolkitsLoader: return (face_toolkits, ) -class CropPortrait: - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "image": ("IMAGE",), - "face_toolkits": ("FACE_TOOLKITS",), - } - } - - RETURN_TYPES = ("IMAGE",) - FUNCTION = "crop_portrait" - CATEGORY = "hellomeme" - - def crop_portrait(self, image, face_toolkits): - image_np = cv2.cvtColor((image[0] * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) - - if min(image_np.shape[:2]) < 512: - raise Exception(f'Image size is too small -> min{image_np.shape[:2]} < 512') - - face_toolkits['face_aligner'].reset_track() - faces = face_toolkits['face_aligner'].forward(image_np) - if len(faces) > 0: - face = sorted(faces, key=lambda x: (x['face_rect'][2] - x['face_rect'][0]) * ( - x['face_rect'][3] - x['face_rect'][1]))[-1] - ref_landmark = face['pre_kpt_222'] - - new_image, new_landmark = crop_and_resize(image_np[np.newaxis, :,:,:], ref_landmark[np.newaxis, :,:], 512, crop=True) - # for x, y in new_landmark[0]: - # cv2.circle(new_image[0], (int(x), int(y)), 2, (0, 255, 0), -1) - else: - raise Exception('No face detected') - new_image = cv2.cvtColor(new_image[0], cv2.COLOR_RGB2BGR) - return (torch.from_numpy(new_image[np.newaxis, :,:,:]).float() / 255., ) - - class GetFaceLandmarks: @classmethod def INPUT_TYPES(s): @@ -276,16 +239,14 @@ class GetFaceLandmarks: def get_face_landmarks(self, face_toolkits, images): frame_list = [cv2.cvtColor((frame * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) for frame in images] frame_num = len(frame_list) - if frame_num == 0: - raise Exception('No image detected') + assert frame_num > 0, 'No image detected' _, landmark_list = det_landmarks(face_toolkits['face_aligner'], frame_list) - if len(frame_list) != frame_num: - raise Exception('Not all images have face detected!') + assert len(frame_list) == frame_num, 'Not all images have face detected!' return (torch.from_numpy(landmark_list).float(), ) -class GetDrivePose: +class GetHeadPose: @classmethod def INPUT_TYPES(s): return { @@ -293,21 +254,22 @@ class GetDrivePose: "face_toolkits": ("FACE_TOOLKITS",), "images": ("IMAGE",), "landmarks": ("FACELANDMARKS222",), + "crop": ("BOOLEAN", {"default": True, "label_on": "True", "label_off": "False"}), } } - RETURN_TYPES = ("DRIVE_POSE",) - RETURN_NAMES = ("drive_pose",) - FUNCTION = "get_drive_pose" + RETURN_TYPES = ("HEAD_POSE",) + RETURN_NAMES = ("head_pose",) + FUNCTION = "get_head_pose" CATEGORY = "hellomeme" - def get_drive_pose(self, face_toolkits, images, landmarks): + def get_head_pose(self, face_toolkits, images, landmarks, crop): frame_list = [cv2.cvtColor((frame * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) for frame in images] landmarks = landmarks.cpu().numpy() - rot_list, trans_list = get_drive_pose(face_toolkits, frame_list, landmarks, save_size=512) - return (dict(rot=np.stack(rot_list), trans=np.stack(trans_list)), ) + new_frames, new_landmarks, rot_list, trans_list = get_drive_pose(face_toolkits, frame_list, landmarks, save_size=512, crop=crop) + return (dict(rot=np.stack(rot_list), trans=np.stack(trans_list), frame=np.stack(new_frames), landmarks=np.stack(new_landmarks)), ) -class GetDriveExpression: +class GetExpression: @classmethod def INPUT_TYPES(s): return { @@ -317,18 +279,18 @@ class GetDriveExpression: "landmarks": ("FACELANDMARKS222",), } } - RETURN_TYPES = ("DRIVE_EXPRESSION",) - RETURN_NAMES = ("drive_exp",) - FUNCTION = "get_drive_expression" + RETURN_TYPES = ("EXPRESSION",) + RETURN_NAMES = ("expression",) + FUNCTION = "get_expression" CATEGORY = "hellomeme" - def get_drive_expression(self, face_toolkits, images, landmarks): + def get_expression(self, face_toolkits, images, landmarks): frame_list = [cv2.cvtColor((frame * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) for frame in images] landmarks = landmarks.cpu().numpy() exp_dict = get_drive_expression(face_toolkits, frame_list, landmarks) return (exp_dict, ) -class GetDriveExpression2: +class GetExpression2: @classmethod def INPUT_TYPES(s): return { @@ -338,11 +300,11 @@ class GetDriveExpression2: "landmarks": ("FACELANDMARKS222",), } } - RETURN_TYPES = ("DRIVE_EXPRESSION2",) - RETURN_NAMES = ("drive_exp2",) - FUNCTION = "get_drive_expression" + RETURN_TYPES = ("EXPRESSION",) + RETURN_NAMES = ("expression",) + FUNCTION = "get_expression" CATEGORY = "hellomeme" - def get_drive_expression(self, face_toolkits, images, landmarks): + def get_expression(self, face_toolkits, images, landmarks): frame_list = [cv2.cvtColor((frame * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) for frame in images] landmarks = landmarks.cpu().numpy() exp_dict = get_drive_expression_pd_fgc(face_toolkits, frame_list, landmarks) @@ -355,9 +317,10 @@ class HMPipelineImage: return { "required": { "hm_image_pipeline": ("HMIMAGEPIPELINE",), - "face_toolkits": ("FACE_TOOLKITS",), - "ref_image": ("IMAGE",), - "drive_pose": ("DRIVE_POSE",), + "ref_head_pose": ("HEAD_POSE",), + "ref_expression": ("EXPRESSION",), + "drive_head_pose": ("HEAD_POSE",), + "drive_expression": ("EXPRESSION",), "trans_ratio": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.1}), "prompt": ("STRING", {"default": DEFAULT_PROMPT}), "negative_prompt": ("STRING", {"default": ''}), @@ -368,10 +331,6 @@ class HMPipelineImage: "guidance_scale": ("FLOAT", {"default": 2.0, "min": 0.0, "max": 100.0, "step": 0.1}), "gpu_id": ("INT", {"default": 0, "min": -1, "max": 16}, ), }, - "optional": { - "drive_exp": ("DRIVE_EXPRESSION", {"default": None},), - "drive_exp2": ("DRIVE_EXPRESSION2", {"default": None},), - } } RETURN_TYPES = ("IMAGE", "LATENT", ) @@ -379,47 +338,30 @@ class HMPipelineImage: CATEGORY = "hellomeme" def sample(self, - hm_image_pipeline, - face_toolkits, - ref_image, - drive_pose, - drive_exp=None, - drive_exp2=None, - trans_ratio='0.0', - prompt=DEFAULT_PROMPT, - negative_prompt='', - steps=25, - seed=0, - guidance_scale=2.0, - gpu_id=0 + hm_image_pipeline, + ref_head_pose, + ref_expression, + drive_head_pose, + drive_expression, + trans_ratio='0.0', + prompt=DEFAULT_PROMPT, + negative_prompt='', + steps=25, + seed=0, + guidance_scale=2.0, + gpu_id=0 ): device = get_torch_device(gpu_id) - image_np = (ref_image[0] * 255).cpu().numpy().astype(np.uint8) - if min(image_np.shape[:2]) < 512: - raise Exception(f'Reference image size is too small -> min{image_np.shape[:2]} < 512') - - image_np = cv2.resize(image_np, (512, 512)) + image_np = cv2.cvtColor(ref_head_pose['frame'][0], cv2.COLOR_BGR2RGB) image_pil = Image.fromarray(image_np) - face_toolkits['face_aligner'].reset_track() - faces = face_toolkits['face_aligner'].forward(image_np) - if len(faces) == 0: raise Exception('No face detected') + ref_trans = ref_head_pose['trans'][0] - face = sorted(faces, key=lambda x: (x['face_rect'][2] - x['face_rect'][0]) * ( - x['face_rect'][3] - x['face_rect'][1]))[-1] - ref_landmark = face['pre_kpt_222'] - - _, ref_trans = face_toolkits['h3dmm'].forward_params(image_np, ref_landmark) - - drive_rot, drive_trans = drive_pose['rot'], drive_pose['trans'] + drive_rot, drive_trans = drive_head_pose['rot'], drive_head_pose['trans'] condition = gen_control_heatmaps(drive_rot, drive_trans, ref_trans, 512, trans_ratio) drive_params = dict(condition=condition.unsqueeze(0).to(dtype=torch.float16, device='cpu')) - - if isinstance(drive_exp, dict): - drive_params.update(drive_exp) - if isinstance(drive_exp2, dict): - drive_params.update(drive_exp2) + drive_params.update(drive_expression) generator = torch.Generator().manual_seed(seed) @@ -444,9 +386,10 @@ class HMPipelineVideo: return { "required":{ "hm_video_pipeline": ("HMVIDEOPIPELINE",), - "face_toolkits": ("FACE_TOOLKITS",), - "ref_image": ("IMAGE",), - "drive_pose": ("DRIVE_POSE",), + "ref_head_pose": ("HEAD_POSE",), + "ref_expression": ("EXPRESSION",), + "drive_head_pose": ("HEAD_POSE",), + "drive_expression": ("EXPRESSION",), "trans_ratio": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.1}), "patch_overlap": ("INT", {"default": 4, "min": 0, "max": 5}), "prompt": ("STRING", {"default": DEFAULT_PROMPT}), @@ -458,10 +401,6 @@ class HMPipelineVideo: "guidance_scale": ("FLOAT", {"default": 2.0, "min": 0.0, "max": 100.0, "step": 0.1}), "gpu_id": ("INT", {"default": 0, "min": -1, "max": 16}, ), }, - "optional": { - "drive_exp": ("DRIVE_EXPRESSION", {"default": None},), - "drive_exp2": ("DRIVE_EXPRESSION2", {"default": None},), - } } RETURN_TYPES = ("IMAGE", "LATENT", ) @@ -470,11 +409,10 @@ class HMPipelineVideo: def sample(self, hm_video_pipeline, - face_toolkits, - ref_image, - drive_pose, - drive_exp=None, - drive_exp2=None, + ref_head_pose, + ref_expression, + drive_head_pose, + drive_expression, trans_ratio=0.0, patch_overlap=4, prompt=DEFAULT_PROMPT, @@ -486,40 +424,26 @@ class HMPipelineVideo: ): device = get_torch_device(gpu_id) - image_np = (ref_image[0] * 255).cpu().numpy().astype(np.uint8) - if min(image_np.shape[:2]) < 512: - raise Exception(f'Reference image size is too small -> min{image_np.shape[:2]} < 512') - - image_np = cv2.resize(image_np, (512, 512)) + image_np = cv2.cvtColor(ref_head_pose['frame'][0], cv2.COLOR_BGR2RGB) image_pil = Image.fromarray(image_np) - face_toolkits['face_aligner'].reset_track() - faces = face_toolkits['face_aligner'].forward(image_np) - face_toolkits['face_aligner'].reset_track() - if len(faces) == 0: raise Exception('No face detected') - - face = sorted(faces, key=lambda x: (x['face_rect'][2] - x['face_rect'][0]) * ( - x['face_rect'][3] - x['face_rect'][1]))[-1] - ref_landmark = face['pre_kpt_222'] - - _, ref_trans = face_toolkits['h3dmm'].forward_params(image_np, ref_landmark) - + ref_rot, ref_trans = ref_head_pose['rot'], ref_head_pose['trans'] generator = torch.Generator().manual_seed(seed) - drive_rot, drive_trans = drive_pose['rot'], drive_pose['trans'] - condition = gen_control_heatmaps(drive_rot, drive_trans, ref_trans, 512, trans_ratio) + drive_rot, drive_trans = drive_head_pose['rot'], drive_head_pose['trans'] + condition = gen_control_heatmaps(drive_rot, drive_trans, ref_trans[0], 512, trans_ratio) + ref_condition = gen_control_heatmaps(ref_rot, ref_trans, ref_trans[0], 512, 0.0) drive_params = dict(condition=condition.unsqueeze(0).to(dtype=torch.float16, device='cpu')) - - if isinstance(drive_exp, dict): - drive_params.update(drive_exp) - if isinstance(drive_exp2, dict): - drive_params.update(drive_exp2) + ref_params = dict(condition=ref_condition.unsqueeze(0).to(dtype=torch.float16, device='cpu')) + drive_params.update(drive_expression) + ref_params.update(ref_expression) res_frames, latents = hm_video_pipeline( prompt=[prompt], strength=1.0, image=image_pil, chunk_overlap=patch_overlap, + ref_params=ref_params, drive_params=drive_params, num_inference_steps=steps, negative_prompt=[negative_prompt], @@ -540,11 +464,10 @@ NODE_CLASS_MAPPINGS = { "HMFaceToolkitsLoader": HMFaceToolkitsLoader, "HMPipelineImage": HMPipelineImage, "HMPipelineVideo": HMPipelineVideo, - "CropPortrait": CropPortrait, "GetFaceLandmarks": GetFaceLandmarks, - "GetDrivePose": GetDrivePose, - "GetDriveExpression": GetDriveExpression, - "GetDriveExpression2": GetDriveExpression2, + "GetHeadPose": GetHeadPose, + "GetExpression": GetExpression, + "GetExpression2": GetExpression2, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -553,9 +476,8 @@ NODE_DISPLAY_NAME_MAPPINGS = { "HMFaceToolkitsLoader": "Load Face Toolkits", "HMPipelineImage": "HelloMeme Image Pipeline", "HMPipelineVideo": "HelloMeme Video Pipeline", - "CropPortrait": "Crop Portrait", "GetFaceLandmarks": "Get Face Landmarks", - "GetDrivePose": "Get Drive Pose", - "GetDriveExpression": "Get Drive Expression", - "GetDriveExpression2": "Get Drive Expression V2", + "GetHeadPose": "Get Head Pose", + "GetExpression": "Get Face Expression", + "GetExpression2": "Get Face Expression V2", } diff --git a/pyproject.toml b/pyproject.toml index df272ec..11ab53f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_hellomeme" description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface" -version = "1.2.2" +version = "1.2.3" license = {file = "LICENSE"} [project.urls]