From 4375948481861242c2cd5745745dc1a381fe6879 Mon Sep 17 00:00:00 2001 From: lipku Date: Fri, 21 Feb 2025 19:52:49 +0800 Subject: [PATCH] add wav2lip256 model --- README.md | 19 +-- app.py | 2 +- wav2lip/models/__init__.py | 4 +- wav2lip/models/conv_384.py | 44 ------- wav2lip/models/syncnet.py | 8 +- wav2lip/models/wav2lip_v2.py | 223 +++++++++++++++++++++++++++++++++++ 6 files changed, 240 insertions(+), 60 deletions(-) delete mode 100644 wav2lip/models/conv_384.py create mode 100644 wav2lip/models/wav2lip_v2.py diff --git a/README.md b/README.md index ba6c7a6..dfc84f1 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ Real time interactive streaming digital human， realize audio video synchronous - 2024.12.8 完善多并发，显存不随并发数增加 - 2024.12.21 添加wav2lip、musetalk模型预热，解决第一次推理卡顿问题。感谢@heimaojinzhangyz - 2024.12.28 添加数字人模型Ultralight-Digital-Human。 感谢@lijihua2017 -- 2025.1.26 添加wav2lip384开源模型 感谢@不蠢不蠢 - 2025.2.7 添加fish-speech tts +- 2025.2.21 添加wav2lip256开源模型 感谢@不蠢不蠢 ## Features 1. 支持多种数字人模型: ernerf、musetalk、wav2lip、Ultralight-Digital-Human @@ -33,10 +33,10 @@ conda activate nerfstream #如果cuda版本不为11.3(运行nvidia-smi确认版本)，根据安装对应版本的pytorch conda install pytorch==1.12.1 torchvision==0.13.1 cudatoolkit=11.3 -c pytorch pip install -r requirements.txt -#如果不训练ernerf模型，不需要安装下面的库 -pip install "git+https://github.com/facebookresearch/pytorch3d.git" -pip install tensorflow-gpu==2.8.0 -pip install --upgrade "protobuf<=3.20.1" +#如果需要训练ernerf模型，安装下面的库 +# pip install "git+https://github.com/facebookresearch/pytorch3d.git" +# pip install tensorflow-gpu==2.8.0 +# pip install --upgrade "protobuf<=3.20.1" ``` 安装常见问题[FAQ](https://livetalking-doc.readthedocs.io/en/latest/faq.html) linux cuda环境搭建可以参考这篇文章 https://zhuanlan.zhihu.com/p/674972886 @@ -46,11 +46,12 @@ linux cuda环境搭建可以参考这篇文章 https://zhuanlan.zhihu.com/p/6749 - 下载模型 百度云盘 密码: ltua GoogleDriver -将wav2lip384.pth拷到本项目的models下, 重命名为wav2lip.pth; -将wav2lip384_avatar1.tar.gz解压后整个文件夹拷到本项目的data/avatars下 +将wav2lip256.pth拷到本项目的models下, 重命名为wav2lip.pth; +将wav2lip256_avatar1.tar.gz解压后整个文件夹拷到本项目的data/avatars下 - 运行 -python app.py --transport webrtc --model wav2lip --avatar_id wav2lip384_avatar1 -用浏览器打开http://serverip:8010/webrtcapi.html , 在文本框输入任意文字，提交。数字人播报该段文字 +python app.py --transport webrtc --model wav2lip --avatar_id wav2lip256_avatar1 +用浏览器打开http://serverip:8010/webrtcapi.html , 先点‘start',播放数字人视频；然后在文本框输入任意文字，提交。数字人播报该段文字 +如果需要商用高清wav2lip模型，可以与我联系购买 服务端需要开放端口 tcp:8010; udp:1-65536 diff --git a/app.py b/app.py index 07b0a4d..2adac66 100644 --- a/app.py +++ b/app.py @@ -478,7 +478,7 @@ if __name__ == '__main__': print(opt) model = load_model("./models/wav2lip.pth") avatar = load_avatar(opt.avatar_id) - warm_up(opt.batch_size,model,384) + warm_up(opt.batch_size,model,256) # for k in range(opt.max_session): # opt.sessionid=k # nerfreal = LipReal(opt,model) diff --git a/wav2lip/models/__init__.py b/wav2lip/models/__init__.py index b0eb492..9f185e6 100644 --- a/wav2lip/models/__init__.py +++ b/wav2lip/models/__init__.py @@ -1,2 +1,2 @@ -from .wav2lip import Wav2Lip #, Wav2Lip_disc_qual -#from .syncnet import SyncNet_color \ No newline at end of file +from .wav2lip_v2 import Wav2Lip, Wav2Lip_disc_qual +from .syncnet import SyncNet_color \ No newline at end of file diff --git a/wav2lip/models/conv_384.py b/wav2lip/models/conv_384.py deleted file mode 100644 index 6fcda7a..0000000 --- a/wav2lip/models/conv_384.py +++ /dev/null @@ -1,44 +0,0 @@ -import torch -from torch import nn -from torch.nn import functional as F - -class Conv2d(nn.Module): - def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs): - super().__init__(*args, **kwargs) - self.conv_block = nn.Sequential( - nn.Conv2d(cin, cout, kernel_size, stride, padding), - nn.BatchNorm2d(cout) - ) - self.act = nn.ReLU() - self.residual = residual - - def forward(self, x): - out = self.conv_block(x) - if self.residual: - out += x - return self.act(out) - -class nonorm_Conv2d(nn.Module): - def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs): - super().__init__(*args, **kwargs) - self.conv_block = nn.Sequential( - nn.Conv2d(cin, cout, kernel_size, stride, padding), - ) - self.act = nn.LeakyReLU(negative_slope=0.01) - - def forward(self, x): - out = self.conv_block(x) - return self.act(out) - -class Conv2dTranspose(nn.Module): - def __init__(self, cin, cout, kernel_size, stride, padding, output_padding=0, *args, **kwargs): - super().__init__(*args, **kwargs) - self.conv_block = nn.Sequential( - nn.ConvTranspose2d(cin, cout, kernel_size, stride, padding, output_padding), - nn.BatchNorm2d(cout) - ) - self.act = nn.ReLU() - - def forward(self, x): - out = self.conv_block(x) - return self.act(out) diff --git a/wav2lip/models/syncnet.py b/wav2lip/models/syncnet.py index e773cdc..15ac672 100644 --- a/wav2lip/models/syncnet.py +++ b/wav2lip/models/syncnet.py @@ -1,7 +1,7 @@ import torch from torch import nn from torch.nn import functional as F - +import pdb from .conv import Conv2d class SyncNet_color(nn.Module): @@ -56,10 +56,10 @@ class SyncNet_color(nn.Module): face_embedding = self.face_encoder(face_sequences) audio_embedding = self.audio_encoder(audio_sequences) - audio_embedding = audio_embedding.view(audio_embedding.size(0), -1) - face_embedding = face_embedding.view(face_embedding.size(0), -1) + audio_embedding = audio_embedding.view(audio_embedding.size(0), -1)#[4, 512] + face_embedding = face_embedding.view(face_embedding.size(0), -1) #[4, 512] - audio_embedding = F.normalize(audio_embedding, p=2, dim=1) + audio_embedding = F.normalize(audio_embedding, p=2, dim=1) #按照宽度方向进行l2归一化 face_embedding = F.normalize(face_embedding, p=2, dim=1) diff --git a/wav2lip/models/wav2lip_v2.py b/wav2lip/models/wav2lip_v2.py new file mode 100644 index 0000000..297a183 --- /dev/null +++ b/wav2lip/models/wav2lip_v2.py @@ -0,0 +1,223 @@ +import torch +from torch import nn +from torch.nn import functional as F +import pdb +from .conv import Conv2dTranspose, Conv2d, nonorm_Conv2d + + +class Wav2Lip(nn.Module): + def __init__(self): + super(Wav2Lip, self).__init__() + + self.face_encoder_blocks = nn.ModuleList([ + nn.Sequential(Conv2d(6, 16, kernel_size=7, stride=1, padding=3)), + + nn.Sequential(Conv2d(16, 32, kernel_size=3, stride=2, padding=1), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True)), + + nn.Sequential(Conv2d(32, 64, kernel_size=3, stride=2, padding=1), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True)), + + nn.Sequential(Conv2d(64, 128, kernel_size=3, stride=2, padding=1), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True)), + + nn.Sequential(Conv2d(128, 256, kernel_size=3, stride=2, padding=1), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True)), + + nn.Sequential(Conv2d(256, 512, kernel_size=3, stride=2, padding=1), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True), ), + + nn.Sequential(Conv2d(512, 512, kernel_size=3, stride=2, padding=1), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True), ), + + nn.Sequential(Conv2d(512, 512, kernel_size=4, stride=1, padding=0), + Conv2d(512, 512, kernel_size=1, stride=1, padding=0)), ]) + + self.audio_encoder = nn.Sequential( + Conv2d(1, 32, kernel_size=3, stride=1, padding=1), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(64, 128, kernel_size=3, stride=3, padding=1), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(256, 512, kernel_size=3, stride=1, padding=0), + Conv2d(512, 512, kernel_size=1, stride=1, padding=0), ) + + self.face_decoder_blocks = nn.ModuleList([ + nn.Sequential(Conv2d(512, 512, kernel_size=1, stride=1, padding=0), ), + + nn.Sequential(Conv2dTranspose(1024, 512, kernel_size=4, stride=1, padding=0), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True), ), + + nn.Sequential(Conv2dTranspose(1024, 512, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True), ), + + nn.Sequential(Conv2dTranspose(1024, 512, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(512, 512, kernel_size=3, stride=1, padding=1, residual=True), ), + + nn.Sequential(Conv2dTranspose(768, 384, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(384, 384, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(384, 384, kernel_size=3, stride=1, padding=1, residual=True), ), + + nn.Sequential(Conv2dTranspose(512, 256, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), ), + + nn.Sequential(Conv2dTranspose(320, 128, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), ), + + nn.Sequential(Conv2dTranspose(160, 64, kernel_size=3, stride=2, padding=1, output_padding=1), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), ), ]) + + self.output_block = nn.Sequential(Conv2d(80, 32, kernel_size=3, stride=1, padding=1), + nn.Conv2d(32, 3, kernel_size=1, stride=1, padding=0), + nn.Sigmoid()) + + def audio_forward(self, audio_sequences, a_alpha=1.): + audio_embedding = self.audio_encoder(audio_sequences) # B, 512, 1, 1 + if a_alpha != 1.: + audio_embedding *= a_alpha + return audio_embedding + + def inference(self, audio_embedding, face_sequences): + feats = [] + x = face_sequences + for f in self.face_encoder_blocks: + x = f(x) + feats.append(x) + + x = audio_embedding + for f in self.face_decoder_blocks: + x = f(x) + try: + x = torch.cat((x, feats[-1]), dim=1) + except Exception as e: + print(x.size()) + print(feats[-1].size()) + raise e + + feats.pop() + + x = self.output_block(x) + outputs = x + + return outputs + + def forward(self, audio_sequences, face_sequences, a_alpha=1.): + # audio_sequences = (B, T, 1, 80, 16) + B = audio_sequences.size(0) + + input_dim_size = len(face_sequences.size()) + if input_dim_size > 4: + audio_sequences = torch.cat([audio_sequences[:, i] for i in range(audio_sequences.size(1))], dim=0)#[bz, 5, 1, 80, 16]->[bz*5, 1, 80, 16] + face_sequences = torch.cat([face_sequences[:, :, i] for i in range(face_sequences.size(2))], dim=0)#[bz, 6, 5, 256, 256]->[bz*5, 6, 256, 256] + + audio_embedding = self.audio_encoder(audio_sequences) # [bz*5, 1, 80, 16]->[bz*5, 512, 1, 1] + if a_alpha != 1.: + audio_embedding *= a_alpha #放大音频强度 + + feats = [] + x = face_sequences + for f in self.face_encoder_blocks: + x = f(x) + feats.append(x) + + x = audio_embedding + for f in self.face_decoder_blocks: + x = f(x) + try: + x = torch.cat((x, feats[-1]), dim=1) + except Exception as e: + print(x.size()) + print(feats[-1].size()) + raise e + + feats.pop() + + x = self.output_block(x) #[bz*5, 80, 256, 256]->[bz*5, 3, 256, 256] + + if input_dim_size > 4: #[bz*5, 3, 256, 256]->[B, 3, 5, 256, 256] + x = torch.split(x, B, dim=0) + outputs = torch.stack(x, dim=2) + + else: + outputs = x + + return outputs + + +class Wav2Lip_disc_qual(nn.Module): + def __init__(self): + super(Wav2Lip_disc_qual, self).__init__() + + self.face_encoder_blocks = nn.ModuleList([ + nn.Sequential(nonorm_Conv2d(3, 32, kernel_size=7, stride=1, padding=3)), + + nn.Sequential(nonorm_Conv2d(32, 64, kernel_size=5, stride=(1, 2), padding=2), + nonorm_Conv2d(64, 64, kernel_size=5, stride=1, padding=2)), + + nn.Sequential(nonorm_Conv2d(64, 128, kernel_size=5, stride=2, padding=2), + nonorm_Conv2d(128, 128, kernel_size=5, stride=1, padding=2)), + + nn.Sequential(nonorm_Conv2d(128, 256, kernel_size=5, stride=2, padding=2), + nonorm_Conv2d(256, 256, kernel_size=5, stride=1, padding=2)), + + nn.Sequential(nonorm_Conv2d(256, 512, kernel_size=3, stride=2, padding=1), + nonorm_Conv2d(512, 512, kernel_size=3, stride=1, padding=1)), + + nn.Sequential(nonorm_Conv2d(512, 512, kernel_size=3, stride=2, padding=1), + nonorm_Conv2d(512, 512, kernel_size=3, stride=1, padding=1), ), + + nn.Sequential(nonorm_Conv2d(512, 512, kernel_size=3, stride=2, padding=1), + nonorm_Conv2d(512, 512, kernel_size=3, stride=1, padding=1), ), + + nn.Sequential(nonorm_Conv2d(512, 512, kernel_size=4, stride=1, padding=0), + nonorm_Conv2d(512, 512, kernel_size=1, stride=1, padding=0)), ]) + + self.binary_pred = nn.Sequential(nn.Conv2d(512, 1, kernel_size=1, stride=1, padding=0), nn.Sigmoid()) + self.label_noise = .0 + + def get_lower_half(self, face_sequences): #取得输入图片的下半部分。 + return face_sequences[:, :, face_sequences.size(2) // 2:] + + def to_2d(self, face_sequences): #将输入的图片序列连接起来，形成一个二维的tensor。 + B = face_sequences.size(0) + face_sequences = torch.cat([face_sequences[:, :, i] for i in range(face_sequences.size(2))], dim=0) + return face_sequences + + def perceptual_forward(self, false_face_sequences): #前传生成图像 + false_face_sequences = self.to_2d(false_face_sequences) #[bz, 3, 5, 256, 256]->[bz*5, 3, 256, 256] + false_face_sequences = self.get_lower_half(false_face_sequences)#[bz*5, 3, 256, 256]->[bz*5, 3, 128, 256] + + false_feats = false_face_sequences + for f in self.face_encoder_blocks: #[bz*5, 3, 128, 256]->[bz*5, 512, 1, 1] + false_feats = f(false_feats) + + return self.binary_pred(false_feats).view(len(false_feats), -1) #[bz*5, 512, 1, 1]->[bz*5, 1, 1] + + def forward(self, face_sequences): #前传真值图像 + face_sequences = self.to_2d(face_sequences) + face_sequences = self.get_lower_half(face_sequences) + + x = face_sequences + for f in self.face_encoder_blocks: + x = f(x) + + return self.binary_pred(x).view(len(x), -1)