From de0e82b9ade390a03161ee3f285bdbf622a97d04 Mon Sep 17 00:00:00 2001 From: AIFSH <1509359472@qq.com> Date: Wed, 24 Apr 2024 10:59:32 +0800 Subject: [PATCH] add real time node --- __init__.py | 8 +- inference_realtime.py | 277 ++++++++++++++++++ models/put model in here | 0 .../models/__pycache__/unet.cpython-310.pyc | Bin 2017 -> 2015 bytes .../models/__pycache__/vae.cpython-310.pyc | Bin 4876 -> 4874 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 340 -> 338 bytes .../__pycache__/blending.cpython-310.pyc | Bin 1737 -> 2791 bytes .../__pycache__/preprocessing.cpython-310.pyc | Bin 4690 -> 4688 bytes .../utils/__pycache__/utils.cpython-310.pyc | Bin 2119 -> 2117 bytes musetalk/utils/blending.py | 41 +++ .../__pycache__/__init__.cpython-310.pyc | Bin 399 -> 397 bytes .../__pycache__/api.cpython-310.pyc | Bin 8314 -> 8312 bytes .../__pycache__/models.cpython-310.pyc | Bin 7167 -> 7165 bytes .../__pycache__/utils.cpython-310.pyc | Bin 9930 -> 9928 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 235 -> 233 bytes .../__pycache__/core.cpython-310.pyc | Bin 4886 -> 4884 bytes .../sfd/__pycache__/__init__.cpython-310.pyc | Bin 260 -> 258 bytes .../sfd/__pycache__/bbox.cpython-310.pyc | Bin 4265 -> 4263 bytes .../sfd/__pycache__/detect.cpython-310.pyc | Bin 3803 -> 3801 bytes .../sfd/__pycache__/net_s3fd.cpython-310.pyc | Bin 3916 -> 3914 bytes .../__pycache__/sfd_detector.cpython-310.pyc | Bin 2962 -> 2960 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 2159 -> 2157 bytes .../__pycache__/model.cpython-310.pyc | Bin 8363 -> 8361 bytes .../__pycache__/resnet.cpython-310.pyc | Bin 3644 -> 3642 bytes .../__pycache__/audio2feature.cpython-310.pyc | Bin 3543 -> 3541 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 5295 -> 5293 bytes .../whisper/__pycache__/audio.cpython-310.pyc | Bin 4014 -> 4012 bytes .../__pycache__/decoding.cpython-310.pyc | Bin 25757 -> 25755 bytes .../whisper/__pycache__/model.cpython-310.pyc | Bin 11474 -> 11472 bytes .../__pycache__/tokenizer.cpython-310.pyc | Bin 10046 -> 10044 bytes .../__pycache__/transcribe.cpython-310.pyc | Bin 9064 -> 9062 bytes .../whisper/__pycache__/utils.cpython-310.pyc | Bin 2879 -> 2877 bytes nodes.py | 41 +++ 33 files changed, 364 insertions(+), 3 deletions(-) create mode 100644 inference_realtime.py delete mode 100644 models/put model in here diff --git a/__init__.py b/__init__.py index 460f42b..96bd5de 100644 --- a/__init__.py +++ b/__init__.py @@ -1,4 +1,4 @@ -from .nodes import MuseTalk,LoadVideo,PreViewVideo,CombineAudioVideo +from .nodes import MuseTalk,LoadVideo,PreViewVideo,CombineAudioVideo,MuseTalkRealTime WEB_DIRECTORY = "./web" # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique @@ -6,7 +6,8 @@ NODE_CLASS_MAPPINGS = { "MuseTalk": MuseTalk, "LoadVideo": LoadVideo, "PreViewVideo": PreViewVideo, - "CombineAudioVideo": CombineAudioVideo + "CombineAudioVideo": CombineAudioVideo, + "MuseTalkRealTime": MuseTalkRealTime } # A dictionary that contains the friendly/humanly readable titles for the nodes @@ -14,5 +15,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "MuseTalk": "MuseTalk Node", "LoadVideo": "Video Loader", "PreViewVideo": "PreView Video", - "CombineAudioVideo": "Combine Audio Video" + "CombineAudioVideo": "Combine Audio Video", + "MuseTalkRealTime": "MuseTalk RealTime Node" } diff --git a/inference_realtime.py b/inference_realtime.py new file mode 100644 index 0000000..9588a2a --- /dev/null +++ b/inference_realtime.py @@ -0,0 +1,277 @@ + +import os +import sys +import cv2 +import json +import torch +import shutil +import pickle +import glob,time +import queue,copy +import threading +from tqdm import tqdm +import numpy as np +import folder_paths +from cuda_malloc import cuda_malloc_supported +from typing import Any +from .musetalk.utils.utils import load_all_model,datagen +from .musetalk.utils.preprocessing import read_imgs,get_landmark_and_bbox +from .musetalk.utils.blending import get_image,get_image_prepare_material,get_image_blending + +parent_directory = os.path.dirname(os.path.abspath(__file__)) +# load model weights +audio_processor,vae,unet,pe = load_all_model(os.path.join(parent_directory,"models")) +device = torch.device("cuda" if cuda_malloc_supported() else "cpu") +timesteps = torch.tensor([0], device=device) + +output_path = folder_paths.get_output_directory() +musetalk_out_path = os.path.join(output_path,"musetalk_realtime") +os.makedirs(musetalk_out_path, exist_ok=True) + +def osmakedirs(path_list): + for path in path_list: + os.makedirs(path) if not os.path.exists(path) else None + +def video2imgs(vid_path, save_path, ext = '.png',cut_frame = 10000000): + cap = cv2.VideoCapture(vid_path) + count = 0 + while True: + if count > cut_frame: + break + ret, frame = cap.read() + if ret: + cv2.imwrite(f"{save_path}/{count:08d}.png", frame) + count += 1 + else: + break + +@torch.no_grad() +class Avatar: + def __init__(self, avatar_id, video_path, bbox_shift, batch_size, preparation): + self.avatar_id = avatar_id + self.video_path = video_path + self.bbox_shift = bbox_shift + self.avatar_path = os.path.join(musetalk_out_path,avatar_id) + self.full_imgs_path = f"{self.avatar_path}/full_imgs" + self.coords_path = f"{self.avatar_path}/coords.pkl" + self.latents_out_path= f"{self.avatar_path}/latents.pt" + self.video_out_path = output_path + self.mask_out_path =f"{self.avatar_path}/mask" + self.mask_coords_path =f"{self.avatar_path}/mask_coords.pkl" + self.avatar_info_path = f"{self.avatar_path}/avator_info.json" + self.avatar_info = { + "avatar_id":avatar_id, + "video_path":video_path, + "bbox_shift":bbox_shift + } + self.preparation = preparation + self.batch_size = batch_size + self.idx = 0 + self.init() + + def init(self): + if self.preparation: + if os.path.exists(self.avatar_path): + response = input(f"{self.avatar_id} exists, Do you want to re-create it ? (y/n)") + if response.lower() == "y": + shutil.rmtree(self.avatar_path) + print("*********************************") + print(f" creating avator: {self.avatar_id}") + print("*********************************") + osmakedirs([self.avatar_path,self.full_imgs_path,self.video_out_path,self.mask_out_path]) + self.prepare_material() + else: + self.input_latent_list_cycle = torch.load(self.latents_out_path) + with open(self.coords_path, 'rb') as f: + self.coord_list_cycle = pickle.load(f) + input_img_list = glob.glob(os.path.join(self.full_imgs_path, '*.[jpJP][pnPN]*[gG]')) + input_img_list = sorted(input_img_list, key=lambda x: int(os.path.splitext(os.path.basename(x))[0])) + self.frame_list_cycle = read_imgs(input_img_list) + with open(self.mask_coords_path, 'rb') as f: + self.mask_coords_list_cycle = pickle.load(f) + input_mask_list = glob.glob(os.path.join(self.mask_out_path, '*.[jpJP][pnPN]*[gG]')) + input_mask_list = sorted(input_mask_list, key=lambda x: int(os.path.splitext(os.path.basename(x))[0])) + self.mask_list_cycle = read_imgs(input_mask_list) + else: + print("*********************************") + print(f" creating avator: {self.avatar_id}") + print("*********************************") + osmakedirs([self.avatar_path,self.full_imgs_path,self.video_out_path,self.mask_out_path]) + self.prepare_material() + else: + with open(self.avatar_info_path, "r") as f: + avatar_info = json.load(f) + + if avatar_info['bbox_shift'] != self.avatar_info['bbox_shift']: + response = input(f" 【bbox_shift】 is changed, you need to re-create it ! (c/continue)") + if response.lower() == "c": + shutil.rmtree(self.avatar_path) + print("*********************************") + print(f" creating avator: {self.avatar_id}") + print("*********************************") + osmakedirs([self.avatar_path,self.full_imgs_path,self.video_out_path,self.mask_out_path]) + self.prepare_material() + else: + sys.exit() + else: + self.input_latent_list_cycle = torch.load(self.latents_out_path) + with open(self.coords_path, 'rb') as f: + self.coord_list_cycle = pickle.load(f) + input_img_list = glob.glob(os.path.join(self.full_imgs_path, '*.[jpJP][pnPN]*[gG]')) + input_img_list = sorted(input_img_list, key=lambda x: int(os.path.splitext(os.path.basename(x))[0])) + self.frame_list_cycle = read_imgs(input_img_list) + with open(self.mask_coords_path, 'rb') as f: + self.mask_coords_list_cycle = pickle.load(f) + input_mask_list = glob.glob(os.path.join(self.mask_out_path, '*.[jpJP][pnPN]*[gG]')) + input_mask_list = sorted(input_mask_list, key=lambda x: int(os.path.splitext(os.path.basename(x))[0])) + self.mask_list_cycle = read_imgs(input_mask_list) + + def prepare_material(self): + print("preparing data materials ... ...") + with open(self.avatar_info_path, "w") as f: + json.dump(self.avatar_info, f) + + if os.path.isfile(self.video_path): + video2imgs(self.video_path, self.full_imgs_path, ext = 'png') + else: + print(f"copy files in {self.video_path}") + files = os.listdir(self.video_path) + files.sort() + files = [file for file in files if file.split(".")[-1]=="png"] + for filename in files: + shutil.copyfile(f"{self.video_path}/{filename}", f"{self.full_imgs_path}/{filename}") + input_img_list = sorted(glob.glob(os.path.join(self.full_imgs_path, '*.[jpJP][pnPN]*[gG]'))) + + print("extracting landmarks...") + coord_list, frame_list = get_landmark_and_bbox(input_img_list, self.bbox_shift) + input_latent_list = [] + idx = -1 + # maker if the bbox is not sufficient + coord_placeholder = (0.0,0.0,0.0,0.0) + for bbox, frame in zip(coord_list, frame_list): + idx = idx + 1 + if bbox == coord_placeholder: + continue + x1, y1, x2, y2 = bbox + crop_frame = frame[y1:y2, x1:x2] + resized_crop_frame = cv2.resize(crop_frame,(256,256),interpolation = cv2.INTER_LANCZOS4) + latents = vae.get_latents_for_unet(resized_crop_frame) + input_latent_list.append(latents) + + self.frame_list_cycle = frame_list + frame_list[::-1] + self.coord_list_cycle = coord_list + coord_list[::-1] + self.input_latent_list_cycle = input_latent_list + input_latent_list[::-1] + self.mask_coords_list_cycle = [] + self.mask_list_cycle = [] + + for i,frame in enumerate(tqdm(self.frame_list_cycle)): + cv2.imwrite(f"{self.full_imgs_path}/{str(i).zfill(8)}.png",frame) + + face_box = self.coord_list_cycle[i] + mask,crop_box = get_image_prepare_material(frame,face_box) + cv2.imwrite(f"{self.mask_out_path}/{str(i).zfill(8)}.png",mask) + self.mask_coords_list_cycle += [crop_box] + self.mask_list_cycle.append(mask) + + with open(self.mask_coords_path, 'wb') as f: + pickle.dump(self.mask_coords_list_cycle, f) + + with open(self.coords_path, 'wb') as f: + pickle.dump(self.coord_list_cycle, f) + + torch.save(self.input_latent_list_cycle, os.path.join(self.latents_out_path)) + # + + def process_frames(self, res_frame_queue,video_len): + print(video_len) + while True: + if self.idx>=video_len-1: + break + try: + start = time.time() + res_frame = res_frame_queue.get(block=True, timeout=1) + except queue.Empty: + continue + + bbox = self.coord_list_cycle[self.idx%(len(self.coord_list_cycle))] + ori_frame = copy.deepcopy(self.frame_list_cycle[self.idx%(len(self.frame_list_cycle))]) + x1, y1, x2, y2 = bbox + try: + res_frame = cv2.resize(res_frame.astype(np.uint8),(x2-x1,y2-y1)) + except: + continue + mask = self.mask_list_cycle[self.idx%(len(self.mask_list_cycle))] + mask_crop_box = self.mask_coords_list_cycle[self.idx%(len(self.mask_coords_list_cycle))] + #combine_frame = get_image(ori_frame,res_frame,bbox) + combine_frame = get_image_blending(ori_frame,res_frame,bbox,mask,mask_crop_box) + + fps = 1/(time.time()-start+1e-6) + print(f"Displaying the {self.idx}-th frame with FPS: {fps:.2f}") + cv2.imwrite(f"{self.avatar_path}/tmp/{str(self.idx).zfill(8)}.png",combine_frame) + self.idx = self.idx + 1 + + def inference(self, audio_path, out_vid_name, fps): + os.makedirs(self.avatar_path+'/tmp',exist_ok =True) + ############################################## extract audio feature ############################################## + whisper_feature = audio_processor.audio2feat(audio_path) + whisper_chunks = audio_processor.feature2chunks(feature_array=whisper_feature,fps=fps) + ############################################## inference batch by batch ############################################## + video_num = len(whisper_chunks) + print("start inference") + res_frame_queue = queue.Queue() + self.idx = 0 + # # Create a sub-thread and start it + process_thread = threading.Thread(target=self.process_frames, args=(res_frame_queue,video_num)) + process_thread.start() + start_time = time.time() + gen = datagen(whisper_chunks,self.input_latent_list_cycle, self.batch_size) + print(f"processing audio:{audio_path} costs {(time.time() - start_time) * 1000}ms") + start_time = time.time() + res_frame_list = [] + + for i, (whisper_batch,latent_batch) in enumerate(tqdm(gen,total=int(np.ceil(float(video_num)/self.batch_size)))): + start_time = time.time() + tensor_list = [torch.FloatTensor(arr) for arr in whisper_batch] + audio_feature_batch = torch.stack(tensor_list).to(unet.device) # torch, B, 5*N,384 + audio_feature_batch = pe(audio_feature_batch) + + pred_latents = unet.model(latent_batch, timesteps, encoder_hidden_states=audio_feature_batch).sample + recon = vae.decode_latents(pred_latents) + for res_frame in recon: + res_frame_queue.put(res_frame) + # Close the queue and sub-thread after all tasks are completed + process_thread.join() + + if out_vid_name is not None: + # optional + cmd_img2video = f"ffmpeg -y -v warning -r {fps} -f image2 -i {self.avatar_path}/tmp/%08d.png -vcodec libx264 -vf format=rgb24,scale=out_color_matrix=bt709,format=yuv420p -crf 18 {self.avatar_path}/temp.mp4" + print(cmd_img2video) + os.system(cmd_img2video) + + output_vid = os.path.join(self.video_out_path, out_vid_name+".mp4") # on + cmd_combine_audio = f"ffmpeg -y -v warning -i {audio_path} -i {self.avatar_path}/temp.mp4 {output_vid}" + print(cmd_combine_audio) + os.system(cmd_combine_audio) + + os.remove(f"{self.avatar_path}/temp.mp4") + shutil.rmtree(f"{self.avatar_path}/tmp") + print(f"result is save to {output_vid}") + return output_vid + +class Infer_Real_Time: + def __init__(self) -> None: + pass + + def __call__(self, audio_path,video_path, + avatar_id,fps=25,batch_size=4, + preparation=True,bbox_shift=0, + *args: Any, **kwds: Any) -> Any: + + avatar = Avatar( + avatar_id = avatar_id, + video_path = video_path, + bbox_shift = bbox_shift, + batch_size = batch_size, + preparation= preparation) + output_name = os.path.basename(audio_path)[:-4] + return avatar.inference(audio_path,output_name,fps) \ No newline at end of file diff --git a/models/put model in here b/models/put model in here deleted file mode 100644 index e69de29..0000000 diff --git a/musetalk/models/__pycache__/unet.cpython-310.pyc b/musetalk/models/__pycache__/unet.cpython-310.pyc index 40f27ddb360e9082d8682a90b995c0a91ebea20d..f0c09639ac93d3687a39eaacb8cac2aae50ef8dc 100644 GIT binary patch delta 32 mcmaFJf1jT_pO=@50SGpSYi#7c%EacbUzDF;viSwmPZj`^TM8Ti delta 34 ocmcc5|B#r&&+=jG*M0D{fo8XLKrvy*=jG*M00JW=xsBXcSU5cNGb(jUatk&;Wl`k=0ECPPv;Y7A diff --git a/musetalk/utils/__pycache__/__init__.cpython-310.pyc b/musetalk/utils/__pycache__/__init__.cpython-310.pyc index e164621ed6c081ff46b4ff5181a793861cec9bfa..16721801f358932b56f2f2d49c8b4a74abae8e97 100644 GIT binary patch delta 44 ucmcb@bcu;OpO=@50SGpSYfR)`&3J3#c1;10KoN-e<)vSgpIyFd diff --git a/musetalk/utils/__pycache__/blending.cpython-310.pyc b/musetalk/utils/__pycache__/blending.cpython-310.pyc index 522507d922132f51de84387ac757f5bb748e6ac6..bc1704a1b9d2f964c588c79d5962b1fc180502b4 100644 GIT binary patch delta 1264 zcmai!NpI6Y7=~x@B(~!$ZPT;~1=IjlBldkqNE{HTLP&^9RVi{iPDyFwXkrkRJ}ZZ| zN06%gf#k@61L9Y3;L5>&fM39!?{nD2GLav@vv_7a@9VtJf0{0Y`Mk@pybKmwC-VD+ zIli=i{LFk&JKEjl%UNC2IqjjcbzT>C*|MX>Bi1PGZ+y<&KW)M=dbiFOB!4}4Wz~^y zt>-)$EUH^~6ZxXqOXN1;MKd53EazMPTF~r`?NJBgcCH+{1nBu;4B&E6Bg8zk_Q1d<6l2tb1u?2j^idCG6v#}is;2?Lziin-q1qXX_u@~p#0$4){4vsHEal?vB zk%Hn*pT`q0fgK8RvkWreA+xErNw+L~U=p{G(E>_Zj$}GM3zPqNLU0BX8y0S1DlSJh zltGl)4v(jQnWqAkVYW|;YYEdZ1!Y={bW4nIipbU$Em>t{rd16unnY)2SUeLsLpRC| zJ*bR0%)m@bq+6SXS#VnDtT~9MXXX)&s-Y7#CkQ7AS;8p-9`j&H)%xZ%)y@#k63!9M z6D|-g5-t%g6Rr@h60Q-h6K)VDzheqjQd3j zRe0PL8E%U@FYu$>l^)&?@i{TeZEkMixHX0NR!S(oCnUkje-okxAH;m0gTOSTDTy7h zP`l%7fEo%QqYLRwa-0$LMWm`Q#Drlq+@GRB+M$@BXJog+F_vP6PaVwv`AGqL6?hmU?n zrEW=X!RB&CHD*T5$@5s0faF1zGRCOM7OZyeQOp(b#kZI%b;HHtSaF*!B9 zAip@XBr`uRiVMumNi0fFy~UYdlo?-~kywx##g&^_oE@K;o0y&&HF+Ie-0 delta 34 ocmcbha!G|dpO=@50SG*`ua8!UhpO=@50SGpSYi#6JWoC2NFUrp^*=)hg&k6v4Cqa9n^ppO=@50SM$W{3JXH@Ez{mkD5GoH2QV stSIBA$;)JGxR-%+t^^S)ChN+b5||2N_JD}lAY%Vy16GO6^74_40Dy=ti2wiq diff --git a/musetalk/utils/face_detection/__pycache__/models.cpython-310.pyc b/musetalk/utils/face_detection/__pycache__/models.cpython-310.pyc index 1590f924db82d60876a9d602d8f323e1268d05f8..4c6a6e9e356f0b6af88ee84d73fb1489986e320f 100644 GIT binary patch delta 32 mcmexw{@0v4pO=@50SGpSYi#6hWn>H2FUrp^**t?WT?zo5@d=s$ delta 34 ocmexs{@>uM)>C)`G;648vQKJyrV{uTH+9 LIv+?@s09H494{1) delta 59 zcmX@%d&-wPpO=@50SJtgTb$t~FYiOE-qBZ{>ku_VJVYO=d(ALHf8 M*Hq^N$uhMd00*`cVgLXD diff --git a/musetalk/utils/face_detection/detection/__pycache__/__init__.cpython-310.pyc b/musetalk/utils/face_detection/detection/__pycache__/__init__.cpython-310.pyc index 4174f71729ce5027d36d3e64372fb24fc9c69817..5ab3cc23af59da912b1ce423836f6bcf69d30b8c 100644 GIT binary patch delta 29 jcmaFO_>z%3pO=@50SGpSYfR)$WlPpC%Fi#ESWyfBa>NLL delta 31 lcmaFK_?nSBpO=@50SJtgpO=@50SGpSYi#6x#LSkeUzDF;viU3XGcEv^QwmuC delta 34 ocmca9ds~(}pO=@50SJtgjcS?>spO=@50SGpSYi#7c&B&InUzDF;viTij94`QpSP7^A delta 34 ocmX>lcSepopO=@50SJtgl3TF(9OHa80H(AGOaK4? diff --git a/musetalk/utils/face_parsing/__pycache__/model.cpython-310.pyc b/musetalk/utils/face_parsing/__pycache__/model.cpython-310.pyc index 0e9bfa8e162d43e548a9ef29697c4cf30928a95e..644c8ea3b9be90fec50d774ec3cbaf5cc2bbe066 100644 GIT binary patch delta 32 mcmZ4OxYCh3pO=@50SGpSYi#5`$;1|-UzDF;viT0vd07CFdkOde delta 34 ocmZ4KxZ06BpO=@50SJtg90F+}1)c^nh diff --git a/musetalk/utils/face_parsing/__pycache__/resnet.cpython-310.pyc b/musetalk/utils/face_parsing/__pycache__/resnet.cpython-310.pyc index fcb8507cecb5886749ac4059f904bab36da08bff..388bc18dd763e176bd915e200eade63388ad2fe6 100644 GIT binary patch delta 32 mcmdlZvrC3MpO=@50SGpSYi#8H$H*3{UzDF;vRRNxn+E`kObCSl delta 34 ocmdlbvqy$IpO=@50SJtgWMT`{FUrp^*_^}lnThe%<^0Lh%>4L*qWt94;^O?GA_tH%7myTdVo_0IWsxzMU7V3vkXob!k~9SoW*|Zw NM3{pJt;t!uxd1UWFiije delta 201 zcmcaAeO;P6pO=@50SJtgWAg^~LPkc- z%}SioOl+1w~voLXF*UsPldQsxYjVofY6N~|n00<()V5(`p`lt7XuAi@+x Ph=T|-5TQ9agEtodkHRko diff --git a/musetalk/whisper/whisper/__pycache__/__init__.cpython-310.pyc b/musetalk/whisper/whisper/__pycache__/__init__.cpython-310.pyc index f15e9ea13520eb118b0400000d8cecd20a561674..7c9b7f9e4e8552e673885006912a8f23560230b0 100644 GIT binary patch delta 32 mcmZ3lxmJ@qpO=@50SGpSYi#7c!^0M$UzDF;viUvF0d@e6l?mkl delta 34 ocmZ3hxn7eypO=@50SJtgyNYNv(>`$+-oW(yR}c%ug0|W z_sNKlAxjufi!qnthw-g?Uork{EkaRn@xeU3v!L+rzbXPvn&eH+qK#m!!q=(~L=DMAEW+k>-WrG^cAw)1{NsG~dq{1Y*lsYb~m(G&y;lNec)) z$Y2I}rnCqEIe`ZXL4YJzi6+N+$_MO<_IGDRYfh#rp03S^+itzlDA|1|h6x=HM<-I} zg}Tp9vm><_9${PSU(q;n=d^J0-uCZaOiO*B(D4d7uZQSw0l_1oA>t3>^ociyjsp3$G)P@c4g&D`J28J zxqUL`6G&smlYGJ>ajt)N)|BE;cz-zc`DiRY8Sif2Rh{2|)$ufGkhd9==1EGA`4FnJ zd&M)!&Oewnw*ixTm=(5&614OMytX=OCY`YD5|IIy17ebto)C$Cdk1VWEfSVo;8 ziv)m7VG|9iD6+>SiBGeH4;fta{$V}Vc4k(|DQL^$rtNQU&D8_QM=_m@##33~xu;7r zN#Cf!u#2=dxa7g@iq@Wew7(X#4bgV|7w7>G=0+LIDC30oQ>il!fEhp&sq&Ces#c;& zekzMW+-&w>57`#72IB)R?bg9z+-U@Ry_VF;D+XOt)WrwaH!&925;ND+9*Ljso%f+?L}n0o8=Vl25L|oOItupxPm8Vwaaivjb{b(+ zQoB8`cN-x*Q#z? zqK*+pe}bU8)TwUyP7y&t1bd3ei$me9Q?7#_v0_2*KSb~FJ(y*8)2rnaD0G;uX6u0H2@ zQA@A5nr>P4f>~lHW_qpvvzlekI?Oga{z;9<;NzF!cTmH{ObXU`gbYIK|t8Bz`bDA&INTe&Me19lCL4W;7gBCuvp(4wokW0+Gta;Q#;t delta 693 zcmZ9{Ur19?90%}z*SRgXne$H)vxpIhL`g-7Wm(Idsn{53>$-Wl*BzX6w_VO|NfGqt zH3rd-da0lu>!~_VK13iv_z*o53`WvRsLNa^B?twb-wk@`K76?6_dUP!JLkbXJemhz zJ`kweBEN&-T`zntuXn67b7h~MwM2{eV>hVy_0$1rTO>ZFHkx|D)@JmAW>05X!Pc{u zy2E0*OQu}wHv2E~JKJ^B%+IhiM>#_c{%h3C{FKG>hAQ4f>ReF4_x-P635(G%EaUTN z0xDwbK(8W$;NB)!5vjq`AQw-D4#O&zhPvdPwhixxsyH#+r}!%JaGoH3U~!}uHt?Nv zT?Eec$SyG$`vmX>+v7QT#Qk^}*2HZ5o^t3Hb*vEv5g}%YyTme4CTM_Cq5MBujGq?$jQr$1|ym z*K4T#2(@<+-z8-?XBN|K?xDzT(LV&TJR#?QhT8Dd)DiyJzy;NGA|B#&9QOldD-r|J#+Smy}Trfh? WO=AJturwL-IJ<*%)^_YmpZ^0QXwm-w diff --git a/musetalk/whisper/whisper/__pycache__/model.cpython-310.pyc b/musetalk/whisper/whisper/__pycache__/model.cpython-310.pyc index 1d1911cbbee46fb36c56d6c96c7fb4059233cab4..39b7f58a6133b83e3359573d7b4e89bb35a4e8ff 100644 GIT binary patch delta 1670 zcmZ{l{ZCs}7{_;8dKnZ76e_g5Ny}L8%3%7#R76}dBQN!3n{0>^Yi%#3QtoBv-gez) zCccbqQRnoDGsgImp&~lBxF+)@k^Kk8nE1=lm>8oj@egqNi=OYf+lm;Q^wZ})=ef^$ z-p=W_liy5w&Dz@P4dPRph<@q~xz*kM%rx!NR`++FJ9zC}vY4>5bGnsE=5*eiGc)DK zMAk?#eNH!Q?#mj5@`AtBlrg$g9Sx0&ZP*x~0c}7#5T~ovf8Tu{ z#QT8<1h_k++r0L`Y$1`fZLvw#G}wb;L91d>7fZf0z+FtXBcQ3?!z(ncI8it*H zz8#y^sIR}kD~-exNjo*eJw_sB7uhITzAj~)Uh}0|G!4yk=f2b zR5Uvm80uc*51tEv?Df--oL-l{4#Zu7XP|f%m;&|#hpDe6>{}EfSCjMex{+qb>9LkJ zTjEMO+5^m0Vf5t0Ix9gqRlI^w$8X(Napgo ztO?5KCM&r>8C5X0PCtS81fP z7$|oDCx%S}PA^D^=f(F%FQPt1rJ9?QidVk$9!&F8?CMd7-s##JkP2Cfa{c4-N4FH5&d4}bC_exAs z7Gs+Q&&Rear|6g1&V8%GjBiZqDKnilG7Nti>?^q! M!&I;(I@&w_4tEmxM&mCxCWe?8{R2dQ)93qKpmWA1_j6CrInT@a zp66-5oB4i5$<)==Y!jb7ooyd0&GxFP{zS3YSjd*b_8%SYNZmrL5H*qudQOX_^a#1D z{ho*`na-D;xd=U4J-~*kqoK9*R&^(1V|1$~)I2I?kk|dmRik2aI^zY0F%H$dZ;#{JPGqDV5)SvmSdG#*JItA-1i2UmFeiQn4u+1M>?t* z1wKlPt`2sH$fdQ6Bg+Xm1k8$fQZslE-w%)UOz=(^2n+bI3sctn);xGyBS`w=M;rZ1q+t=+~A(>rkNke7V#cn719BhMfjp5>S0+ zMOrTyW<}D_x#`TuQcHTy66^w!Yp^cTA8oiX43v0~aaZ zv6HRPI~~0~>5!U~H6N#+JC?j_NX4a7ZI-Ur=uA%o6*~*t-@PBQTcPW4yiei^uvH2K zpRqD9gL*gP8ZxdUgVuu6>@wX6_OTn(-8H3LMMySn9V?VBb^Ty}L>@-Dj?{JlIYR(s zRqdv8AroIp>HLKFDzsWMvP*{8=uKJ+nS&!plv?jY*vIv`WKMK0I-B9gV>}*BWHR%7 zKW1Y5BfM;;1HK&JpwCpJ9}AlfasE@qdj;GSE?xp4lv=^1;}bYGD6%U*F7=iBl)qf> zA+ieC1x&lR%2G|UD$6rjai@#?2BIGT<$iuj+rue#g_gq)3Vs-Vn4PCz!oyF0fk~?3 sx|WG2({mhu2mEtD0loxkfi|EA*aHj$w*aY$1`BtTU2GIp6lQj3xBFMSZRro%0xdUQ^*PC7fYId@vE zsW$rJPkg&5YeA(5Vu_K|K~%tsC<+QSidDJ(y!k+k#xy=?jj89{q3v45CjIu#Ip==o z+;hHr$5W?Lai_F2wg~^c?Kh3Z6Cq8C+R#x=dgnriH2knjTJU4IBtajCBU@T_W2_OB zfi>%-o^i5FMy#~Gt1rEa0l5@RcdP}+am9*gPyBf@K1uL|_gwUyfVM~6o!F@D6T^wd z7iswtf)Rjy_#?wvMwyHtjF9UP!OH{-w!eb5XUKlgyQg$gyG@m< z%b3Yco4aW{!(@=Vph$s&F~PI=BQ-v%`P_=JAiB!>wGnZ-Y*kztUMJFmh$n}di|a}- zhj+S8FS9$%EU2KOio~(xBguam+soUulj70x>KorB(uIDiCd?hT)5?vK|Fk$!zP9}& zE{#BHx~y#w&jGH}od?!GcspE^BSIV$q!OGK%@sG_G0T!@8?+ssXDrk6e-u&M>GY#K zzmnmKTd?$t6-!oMGi@pEam^gVDsy@Mw!g2kGIi?_Twc-CG5nd9r7er-GiKVt?(zTX z1dMrw$|f!RD)EepGmBeLJijcy0mT!m+Jpn@sA}{FbaiZxSG{%LMG7X_zy!f(2r`6I zH}f*OxjnU^qkY5rZSaONyM1|!8M+$13z&D=LY|5o~ZSLwZ2+2 z3x)r!*jcx={XFh7V(>lDg?QdFU6eaRz*hSm*-x&Z#cZ@?vwmV!8@McMgkePdT31*9 zHN}MaK(5QN;23@sClf~fLG8Sls9&>AwU`LuM}*nCd4LQHcad#ev@RV;zDCP1f()9r zi}(G_rOiXClcW_H$%B(;;0{2L0#6XUM{tJVLxPV8RENfB^#y_I@OQNOf#4^CaRSx4 zc~Ec6UUWQ$|9l5RY>^%iJC|+O_KM@nE@`L4)63gheYwr)te^KB=uf4z-0w?eopvGWgITh4julW-7^I0*^6Ag z9=tfK__^V(mN4EVBOrA*&3j=FaVK&pUDGn@CAOdh{mKr9y$2dsYMUcykTDn1HqU_B z#UM&qen-GURaz2{IXyEobLCG29H*KuaVt?8aT&6l2N}Sm_Wl4;{6R3mKz+d8Q)5d$JsG#47Dxs*l9l$IcB4k&C2$?R+aY#^IR3rGZq-=VZ SgR&>uqNb~~1+RPMH`?FT=Ml01 delta 1891 zcmaJ>Yiv|S6z1;U-R{$FTl!GjLTgFAE#>vcm{dt)2tvpbZ49O%mhJA8-t_Lhn=`l7 znwn}f(H|x%LrR6Js1O1{2(A`t5m8VPAAm1#e)-FU_#)9CrY6>N=Ca$hDx2(gXU?4Y z&Y5$*nUl#=$*2>LM;764O>^V1Xe^-hh`PX0ttWpjuwTQ8UDl41-DNSF90+ahcn*CH zzx1tGCH1tEVKQW;?CnFT?F?v1!R(fm10x-7HYb|G1JRviev079{Lb*SPun4qu?^ZT zF%fG{6!(B2k3HgiY~4LC(r`C|;e%cHk--e3ScV@aX=yLP%LEIyzlyeJ$^J_I{`fcA zohn&fMoezn+)deOCjHz6MGEAM0iMT?)c8To<5rA9Y%3quCdK9Q)~GTZBvK*5iLtiQ zykgAZy{|Qei%BdtH@n+(&#J`MdD?0C*QlXQx@y=093tXq)%8uR*Cn(Co4U}Gx zJ8rL)RZBDCM8(Rk0z-L`AqQ;uMjRpHAg3b~Wk%dyxq5AZCDu7=J3Pl&df-MBVcY5T zyz(B$MCBT7N_<**bI1Rt9>+bdnPqs+Y>q3p8BtM{ykkF7T-6W?O)gJa7SX58l!fKw z|5XV%oFA`h(Som2{C(o|;&xQiPm6CwHHE4-V#m6xo4t-*@8jdu$G4p$XVMXTLU0~I z22ciOPKGyiCD%XDwSHYUyrInQP|jk8uCyS7!1$2m!ZBJ375iM$Vp^yYmujlSp_-}I zFL0W~@(6c)E7;EtkmWYl8WPtC0}bYKN99mFYlUQnyUZc7lA*{BwhM z$ova+d4HX!y~lj4{9`i3vbC&)*#eU1f3aZ(ygv^MP1&Wg_(R;*IBCPMfgp>S(oBE!O6WSbUk zjU$NzGz=n0ziGSp6671(##ALqY%-JsC&$1Y0Y5puP4EuEy9DnMyicGibeKk;6Q~Mb zrqOo2tU&WA?BKjxF_E${m0s!foo^!*mQ5ZrlG%2>TZe;!VdC{Wl`6rWzx%S zfe+V}9bU^n)VxgFK&9@X885t){ IzN6Fy0G4AAg8%>k diff --git a/musetalk/whisper/whisper/__pycache__/utils.cpython-310.pyc b/musetalk/whisper/whisper/__pycache__/utils.cpython-310.pyc index 35460aeff19193061a8d652e67ac3fefd7a71e7c..0d01ecd2dd8927ae6d80c5b2c411455df79227b0 100644 GIT binary patch delta 282 zcmdllwpWZhpO=@50SGpSYi#5;V`K}|FUrp^+3d;4$HI7Pas+D@qt4{RtjUaelhxT| z*>r&_i}WUYv6(TNPHtrD0t)l6Ybxjig*91<3_-FwAi@Yp++r_EEyzhsPAxK?9LjFZ zXuEkLdn8l68A#p|M1WW!zZi8inTnJ^d}R<}4k9c-gdB*l0uk0A0%V7uChINE)V$K% z)S|?aR5bgFY=A0&u7J1^$iBsxS!4pz$^s|Y8760P88SLep3UVa26E{wmXyrok|I-( N3R@82H2FW50{~F$I#&Py delta 298 zcmdlhwqJ}ppO=@50SJtg(a!XH@EztfWM%*C#$pa&Gz zWGONL$!dcLLm+XBy(qOHCows-$Y`=5v+(3Jb`3_G&He0=Ou42YH5MQO#0vSvsH4eL zqy*wCg9tMaVGbhXK!hcTumTYvd;Bz6Z*ivPmFA`vC6=V3*;!-_Q~`7e#GOF)Eym2r of$TDq3%S%89VgG^@)HBO?-ol+W^zf936NQ210ozJ|K)N30J4=r>;M1& diff --git a/nodes.py b/nodes.py index acf41e4..05a42a7 100644 --- a/nodes.py +++ b/nodes.py @@ -1,6 +1,7 @@ import os import folder_paths from .inference import MuseTalk_INFER +from .inference_realtime import Infer_Real_Time from pydub import AudioSegment from moviepy.editor import VideoFileClip,AudioFileClip @@ -8,6 +9,46 @@ parent_directory = os.path.dirname(os.path.abspath(__file__)) input_path = folder_paths.get_input_directory() out_path = folder_paths.get_output_directory() +class MuseTalkRealTime: + @classmethod + def INPUT_TYPES(s): + return { + "required":{ + "audio":("AUDIO",), + "video":("VIDEO",), + "avatar_id":("STRING",{ + "default": "talker1" + }), + "bbox_shift":("INT",{ + "default":0 + }), + "fps":("INT",{ + "default":25 + }), + "batch_size":("INT",{ + "default":4 + }), + "preparation":("BOOLEAN",{ + "default":True + }) + } + } + CATEGORY = "AIFSH_MuseTalk" + DESCRIPTION = "hello world!" + + RETURN_TYPES = ("VIDEO",) + + OUTPUT_NODE = False + + FUNCTION = "process" + + def process(self,audio,video,avatar_id,bbox_shift,fps,batch_size,preparation): + muse_talk_real_time = Infer_Real_Time() + output_vid_name = muse_talk_real_time(audio, video,avatar_id,fps=fps,batch_size=batch_size, + preparation=preparation,bbox_shift=bbox_shift) + return (output_vid_name,) + + class MuseTalk: @classmethod def INPUT_TYPES(s):