improve musetalk quality

README.md

@@ -1,15 +1,16 @@
 A streaming digital human based on the Ernerf model， realize audio video synchronous dialogue. It can basically achieve commercial effects.  
 基于ernerf模型的流式数字人，实现音视频同步对话。基本可以达到商用效果
 
-[效果演示](https://www.bilibili.com/video/BV1PM4m1y7Q2/)
+[ernerf效果](https://www.bilibili.com/video/BV1PM4m1y7Q2/)  [musetalk效果](https://www.bilibili.com/video/BV1gm421N7vQ/)
 
 ## Features
-1. 支持声音克隆
+1. 支持多种数字人模型: ernerf、musetalk
-2. 支持大模型对话
+2. 支持声音克隆
 3. 支持多种音频特征驱动：wav2vec、hubert
 4. 支持全身视频拼接
 5. 支持rtmp和webrtc
 6. 支持视频编排：不说话时播放自定义视频
+7. 支持大模型对话
 
 ## 1. Installation
 
@@ -205,7 +206,8 @@ docker版本已经不是最新代码，可以作为一个空环境，把最新
 - [x] 添加chatgpt实现数字人对话
 - [x] 声音克隆
 - [x] 数字人静音时用一段视频代替
-- [ ] MuseTalk
+- [x] MuseTalk
+- [ ] SyncTalk
 
 如果本项目对你有帮助，帮忙点个star。也欢迎感兴趣的朋友一起来完善该项目。  
 Email: lipku@foxmail.com  

musereal.py

@@ -101,6 +101,19 @@ class MuseReal:
         #               self.batch_size)
         self.asr.run_step()
         whisper_chunks = self.asr.get_next_feat()
+        is_all_silence=True
+        audio_frames = []
+        for _ in range(self.batch_size*2):
+            frame,type = self.asr.get_audio_out()
+            audio_frames.append((frame,type))
+            if type==0:
+                is_all_silence=False
+        if is_all_silence:
+            for i in range(self.batch_size):
+                self.res_frame_queue.put((None,self.__mirror_index(self.idx),audio_frames[i*2:i*2+2]))
+                self.idx = self.idx + 1
+        else:
+            print('infer=======')
             whisper_batch = np.stack(whisper_chunks)
             latent_batch = []
             for i in range(self.batch_size):
@@ -121,9 +134,9 @@ class MuseReal:
                                         encoder_hidden_states=audio_feature_batch).sample
             recon = self.vae.decode_latents(pred_latents)
             #print('diffusion len=',len(recon))
-        for res_frame in recon:
+            for i,res_frame in enumerate(recon):
                 #self.__pushmedia(res_frame,loop,audio_track,video_track)
-            self.res_frame_queue.put((res_frame,self.__mirror_index(self.idx)))
+                self.res_frame_queue.put((res_frame,self.__mirror_index(self.idx),audio_frames[i*2:i*2+2]))
                 self.idx = self.idx + 1
       
 
@@ -131,9 +144,12 @@ class MuseReal:
         
         while not quit_event.is_set():
             try:
-                res_frame,idx = self.res_frame_queue.get(block=True, timeout=1)
+                res_frame,idx,audio_frames = self.res_frame_queue.get(block=True, timeout=1)
             except queue.Empty:
                 continue
+            if audio_frames[0][1]==1 and audio_frames[1][1]==1: #全为静音数据，只需要取fullimg
+                combine_frame = self.frame_list_cycle[idx]
+            else:
                 bbox = self.coord_list_cycle[idx]
                 ori_frame = copy.deepcopy(self.frame_list_cycle[idx])
                 x1, y1, x2, y2 = bbox
@@ -150,10 +166,8 @@ class MuseReal:
             new_frame = VideoFrame.from_ndarray(image, format="bgr24")
             asyncio.run_coroutine_threadsafe(video_track._queue.put(new_frame), loop) 
 
-            audiotype = 0
+            for audio_frame in audio_frames:
-            for _ in range(2):
+                frame,type = audio_frame
-                frame,type = self.asr.get_audio_out()
-                audiotype += type
                 frame = (frame * 32767).astype(np.int16)
                 new_frame = AudioFrame(format='s16', layout='mono', samples=frame.shape[0])
                 new_frame.planes[0].update(frame.tobytes())
@@ -185,9 +199,10 @@ class MuseReal:
                 print(f"------actual avg infer fps:{count/totaltime:.4f}")
                 count=0
                 totaltime=0
-            if self.res_frame_queue.qsize()>2*self.opt.batch_size:
+            if video_track._queue.qsize()>=2*self.opt.batch_size:
-                time.sleep(0.1)
+                #print('sleep qsize=',video_track._queue.qsize())
-                #print('sleep')
+                time.sleep(0.04*self.opt.batch_size*1.5)
+                
             # delay = _starttime+_totalframe*0.04-time.perf_counter() #40ms
             # if delay > 0:
             #     time.sleep(delay)