Yaxin
/
speech-demo-service
31
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

添加对话列表

Browse Source
main
xueqingkun 2 months ago
parent 139c11cb1c
commit 7d0d30b9c6
6 changed files with 330 additions and 15 deletions
Show Stats Download Patch File Download Diff File

4
src/main/java/com/supervision/SpeechDemoServiceApplication.java
Unescape Escape View File

@ -1,5 +1,6 @@
package com.supervision;
import com.supervision.util.PredefinedAnswerAssistant;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
@ -11,6 +12,9 @@ public class SpeechDemoServiceApplication {
public static void main(String[] args) {
SpringApplication.run(SpeechDemoServiceApplication.class, args);
// 加载预定义回答
PredefinedAnswerAssistant assistant = new PredefinedAnswerAssistant();
assistant.loadAnswer();
}
@Bean
public RestTemplate restTemplate() {

68
src/main/java/com/supervision/service/impl/ChatServiceImpl.java
Unescape Escape View File

@ -2,9 +2,12 @@ package com.supervision.service.impl;
import cn.hutool.core.codec.Base64;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.TimeInterval;
import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.StrUtil;
import cn.hutool.crypto.digest.MD5;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSON;
import com.supervision.dto.dify.ChatResDTO;
import com.supervision.dto.paddlespeech.res.TtsResultDTO;
@ -16,10 +19,7 @@ import com.supervision.model.dify.DIFYChatReqInputVO;
import com.supervision.model.dify.DifyChatReqVO;
import com.supervision.model.dify.StreamResponse;
import com.supervision.service.IChatService;
import com.supervision.util.AsrUtil;
import com.supervision.util.DifyApiUtil;
import com.supervision.util.TtsUtil;
import com.supervision.util.WavUtil;
import com.supervision.util.*;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -36,8 +36,10 @@ import org.springframework.util.StopWatch;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.io.IOException;
import java.time.Duration;
import java.util.*;
import java.util.HashMap;
import java.util.Map;
@ -54,9 +56,12 @@ public class ChatServiceImpl implements IChatService {
@Value("${dify.url}")
private String difyUrl;
@Value("${dify.app-auth}")
private String difyAppAuth;
private final WebClient webClient;
private final DifyApiUtil difyApiUtil;
@ -83,7 +88,12 @@ public class ChatServiceImpl implements IChatService {
StringBuilder sentence = new StringBuilder();
log.info("query:{}", query);
return webClient.post()
PredefinedAnswerAssistant predefinedAnswerAssistant = new PredefinedAnswerAssistant();
TimeInterval timeInterval = DateUtil.timer();
timeInterval.start();
timeInterval.start("start");
log.info("request:时间：{}", DateUtil.now());
Flux<StreamResponse> flux = webClient.post()
.uri(difyUrl)
.headers(httpHeaders -> {
httpHeaders.setContentType(MediaType.APPLICATION_JSON);
@ -91,28 +101,41 @@ public class ChatServiceImpl implements IChatService {
})
.bodyValue(JSON.toJSONString(difyChatReqVO))
.retrieve()
.bodyToFlux(StreamResponse.class)
.mapNotNull(response -> {
.bodyToFlux(StreamResponse.class);
Flux<StreamResponse> timeoutMono1 = Flux.interval(Duration.ofMillis(1000)).take(8).map(aLong -> {
StreamResponse timeoutResponse = new StreamResponse();
timeoutResponse.setEvent("timeout");
return timeoutResponse;
});
Flux<StreamResponse> mergedFlux = flux.mergeWith(timeoutMono1);
return mergedFlux.mapNotNull(response -> {
log.info("response:时间：{},响应回答：{}", timeInterval.intervalMs(), JSONUtil.toJsonStr(response));
Map<String, String> map = new HashMap<>();
map.put("event", response.getEvent());
if (response.getEvent().equals("message") && response.getAnswer() != null) {
log.info("response:时间：{}", DateUtil.now());
log.info("response:{}", response);
//遍历answer中的每一个字符，判断是否为标点符号，如果是，说明是句子的结尾，将标点符号前的文本拼接到sentence中，并打印，然后清空sentence，如果标点符号后还有文本，将文本拼接到sentence中
for (char ch : response.getAnswer().toCharArray()) {
sentence.append(ch);
fullAskString.append(ch);
if (ch == '。' || ch == '！' || ch == '？' || ch == '，' || ch == '、' || ch == '‘' || ch == '’' || ch == '“' || ch == '”') { // Check for punctuation marks
log.info(sentence.toString());
log.info("响应回答:{}", sentence);
TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(sentence.toString());
String voiceBaseId = UUID.randomUUID().toString();
audioList.add(voiceBaseId);
audioCache.put(voiceBaseId, ttsResultDTO.getAudio());
map.put("audioId", voiceBaseId);
sentence.setLength(0);
predefinedAnswerAssistant.closePreviewAnswer();
return ServerSentEvent.builder(map).build();
}
}
}
if (response.getEvent().equals("message_end")) {
predefinedAnswerAssistant.closePreviewAnswer();
map.put("sessionId", response.getConversation_id());
if (!sentence.isEmpty()) {
log.info(sentence.toString());
@ -126,7 +149,7 @@ public class ChatServiceImpl implements IChatService {
String uuid = cn.hutool.core.lang.UUID.randomUUID().toString();
List<String> collect = audioList.stream().map(audioCache::get).filter(Objects::nonNull).collect(Collectors.toList());
String fullAnswer = WavUtil.mergeWavFilesToBase64(collect);
audioCache.put(uuid,fullAnswer);
audioCache.put(uuid, fullAnswer);
builder.answerInfo(AnswerInfo.builder().contentType(2).message(fullAskString.toString()).voiceBaseId(uuid).build());
builder.sessionId(response.getConversation_id());
@ -134,6 +157,29 @@ public class ChatServiceImpl implements IChatService {
this.appendDialogCache(response.getConversation_id(), builder.build());
return ServerSentEvent.builder(map).build();
}
if (predefinedAnswerAssistant.overThreshold()) {
if (StrUtil.isEmpty(difyChatReqVO.getConversation_id()) && !predefinedAnswerAssistant.isSkipFirst()) {
predefinedAnswerAssistant.skipFirst();
String text = predefinedAnswerAssistant.getHelloAnswer();
predefinedAnswerAssistant.resetInitTime(text);
log.info("超过阈值，返回预定义打招呼回答:{}", text);
String voiceBaseId = MD5.create().digestHex(text);
map.put("audioId", voiceBaseId);
log.info("response前置回答id：{}", voiceBaseId);
audioCache.put(voiceBaseId, predefinedAnswerAssistant.getHelloAnswerAudio(text));
return ServerSentEvent.builder(map).build();
} else {
String text = predefinedAnswerAssistant.getPredefinedAnswer();
log.info("超过阈值，返回预定义回答:{}", text);
predefinedAnswerAssistant.resetInitTime(text);
String voiceBaseId = MD5.create().digestHex(text);
map.put("audioId", voiceBaseId);
log.info("response前置回答id：{}", voiceBaseId);
audioCache.put(voiceBaseId, predefinedAnswerAssistant.getPredefinedAnswerAudio(text));
return ServerSentEvent.builder(map).build();
}
}
return null;
}).filter(Objects::nonNull);
}
@ -186,7 +232,7 @@ public class ChatServiceImpl implements IChatService {
public void getAudio(HttpServletResponse response, String audioId) throws IOException {
log.info("audioId:{}", audioId);
if (StrUtil.isEmpty(audioId) && StrUtil.equals(audioId, "undefined")) {
if (StrUtil.isEmpty(audioId) || StrUtil.equals(audioId, "undefined")) {
return;
}
Base64.decodeToStream(audioCache.get(audioId), response.getOutputStream(), false);
@ -262,6 +308,8 @@ public class ChatServiceImpl implements IChatService {
return sb.toString();
}
// 示例测试
public static void main(String[] args) {
String sampleText = "欢迎来到孟河小镇，体验独特的小镇风情；另外，还有梦和小镇等待你探访。";

7
src/main/java/com/supervision/util/DifyApiUtil.java
Unescape Escape View File

@ -1,5 +1,6 @@
package com.supervision.util;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSON;
import com.supervision.dto.dify.ChatResDTO;
@ -28,6 +29,8 @@ import reactor.core.publisher.Flux;
import java.nio.charset.StandardCharsets;
import static com.supervision.common.constant.DifyConstants.CHAT_RESPONSE_MODE_BLOCKING;
@Component
@Slf4j
@Service
@ -73,8 +76,10 @@ public class DifyApiUtil {
ChatResDTO execute;
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
HttpPost httpPost = new HttpPost(difyUrl);
httpPost.setHeader(HttpHeaders.AUTHORIZATION, difyAppAuth);
String authorization = StrUtil.startWith("Bearer ", difyAppAuth) ? difyAppAuth : "Bearer " + difyAppAuth;
httpPost.setHeader(HttpHeaders.AUTHORIZATION, authorization);
httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "application/json");
difyChatReqVO.setResponse_mode(CHAT_RESPONSE_MODE_BLOCKING);
log.info("发起对话：{}", difyChatReqVO);
StringEntity entity = new StringEntity(com.alibaba.fastjson.JSONObject.toJSON(difyChatReqVO).toString(), StandardCharsets.UTF_8);
httpPost.setEntity(entity);

151
src/main/java/com/supervision/util/PredefinedAnswerAssistant.java
Unescape Escape View File

@ -0,0 +1,151 @@
package com.supervision.util;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import com.supervision.dto.paddlespeech.res.TtsResultDTO;
import lombok.extern.slf4j.Slf4j;
import java.util.*;
/**
* 预定义答案助手
*/
@Slf4j
public class PredefinedAnswerAssistant {
/**
* 问候答案
*/
private static final Map<String,TtsResultDTO> helloAnswerMap = new HashMap<>();
/**
* 预定义答案
*/
private static final Map<String,TtsResultDTO> predefinedAnswerMap = new HashMap<>();
/**
* 阈值时间，超过这个阈值就开始调换出一个预置回答 单位毫秒 默认1000毫秒
*/
private int thresholdMillis = 500;
/**
* 初始化时间
*/
private DateTime initTime = DateUtil.date().offset(DateField.MILLISECOND, thresholdMillis);
private boolean skip = false;
private boolean skipFirst = false;
/**
* 当前时间是否超过阈值
* @return true 超过阈值 false 未超过阈值
*/
public boolean overThreshold(){
return !skip && initTime.before(new Date());
}
public void skipFirst(){
this.skipFirst = true;
}
public boolean isSkipFirst(){
return skipFirst;
}
/**
* 获取问候答案
* @return 问候答案
*/
public String getHelloAnswer() {
List<String> collect = helloAnswerMap.keySet().stream().toList();
return collect.get(RandomUtil.randomInt(0, collect.size() - 1));
}
public void setThresholdMillis(int thresholdMillis) {
this.thresholdMillis = thresholdMillis;
}
public void closePreviewAnswer(){
skip = true;
}
/**
* 获取问候答案音频
* @param text 问候答案
* @return 问候答案音频
*/
public String getHelloAnswerAudio(String text) {
return helloAnswerMap.get(text).getAudio();
}
/**
* 获取预定义答案
* @return 预定义答案
*/
public String getPredefinedAnswer() {
List<String> collect = predefinedAnswerMap.keySet().stream().toList();
return collect.get(RandomUtil.randomInt(0, collect.size() - 1));
}
public String getPredefinedAnswerAudio(String text) {
return predefinedAnswerMap.get(text).getAudio();
}
public void resetInitTime(String text){
if (StrUtil.isEmpty(text)){
return;
}
TtsResultDTO ttsResultDTO = null == predefinedAnswerMap.get(text) ? helloAnswerMap.get(text) : predefinedAnswerMap.get(text);
if (null == ttsResultDTO){
return;
}
double v = NumberUtil.parseDouble(ttsResultDTO.getDuration()) * 1000;
this.initTime = DateUtil.offsetMillisecond(DateUtil.date(), NumberUtil.parseInt(String.valueOf(v)));
}
/**
* 加载问候答案
*/
public void loadAnswer() {
List<String> helloAnswerList = List.of("您好，我是您的康养顾问小苏，很开心接到您的来电","您好，我是您的康养顾问小苏，感谢您的来电",
"您好，我是您的康养顾问小陈，很高兴接到您的电话");
log.info("开始加载问候答案....");
for (String text : helloAnswerList) {
if (helloAnswerMap.containsKey(text)) {
continue;
}
TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(text);
helloAnswerMap.put(text,ttsResultDTO);
}
log.info("问候答案加载完成....");
List<String> predefinedAnswerList = List.of("您好，我是您的康养顾问小陈，很高兴接到您的电话", "我会结合最新的资料，为您提供最贴心的解答",
"结合最新的信息，为您提供最合适的建议", "结合最新的资料为您提供详细的解答", "根据掌握的高黎贡勐赫小镇信息，为您提供最准确的服务与建议",
"根据最新动态，为您提供最精准的解答与建议", "结合最新情况，为您提供最贴心的服务与支持");
log.info("开始加载预定义答案....");
for (String text : predefinedAnswerList) {
if (predefinedAnswerMap.containsKey(text)){
continue;
}
TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(text);
predefinedAnswerMap.put(text,ttsResultDTO);
}
log.info("预定义答案加载完成....");
}
}

1
src/main/java/com/supervision/util/TtsUtil.java
Unescape Escape View File

@ -31,6 +31,5 @@ public class TtsUtil {
} catch (Exception e) {
throw new BusinessException("语音转换文字失败", e);
}
}
}

114
src/test/java/com/supervision/SpeechDemoServiceApplicationTests.java
Unescape Escape View File

@ -1,18 +1,35 @@
package com.supervision;
import cn.hutool.core.codec.Base64;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSON;
import com.supervision.dto.dify.ChatResDTO;
import com.supervision.dto.paddlespeech.res.TtsResultDTO;
import com.supervision.dto.robot.AnswerInfo;
import com.supervision.dto.robot.AskInfo;
import com.supervision.model.dify.DIFYChatReqInputVO;
import com.supervision.model.dify.DifyChatReqVO;
import com.supervision.model.dify.StreamResponse;
import com.supervision.util.AsrUtil;
import com.supervision.util.DifyApiUtil;
import com.supervision.util.TtsUtil;
import com.supervision.util.WavUtil;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.http.MediaType;
import org.springframework.http.codec.ServerSentEvent;
import org.springframework.web.reactive.function.client.WebClient;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@SpringBootTest
class SpeechDemoServiceApplicationTests {
@ -23,9 +40,29 @@ class SpeechDemoServiceApplicationTests {
@Test
void testTtsTransform() {
TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform("欢迎来电，我是您的康养顾问小苏。关于您的问题，我们已经为您查询到了相关信息。");
System.out.println(JSONUtil.toJsonStr(ttsResultDTO));
// https://www.toolfk.com/zh-cn/tools/base64-to-audio.html base64转音频
String text = """
您好，我是您的康养顾问小苏，很开心接到您的来电
您好，我是您的康养顾问小苏，感谢您的来电
您好，我是您的康养顾问小陈，很高兴接到您的电话
关于您的问题，我已经为您整理好了相关信息
我会结合最新的资料，为您提供最贴心的解答
结合最新的信息，为您提供最合适的建议
结合最新的资料为您提供详细的解答
根据掌握的高黎贡勐赫小镇信息，为您提供最准确的服务与建议
根据最新动态，为您提供最精准的解答与建议
结合最新情况，为您提供最贴心的服务与支持
""";
List<String> collect = Arrays.stream(text.split("\n")).filter(s -> !s.isBlank()).toList();
for (int i = 0; i < collect.size(); i++) {
TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(collect.get(i));
String jsonStr = JSONUtil.toJsonStr(ttsResultDTO);
System.out.println(jsonStr);
// https://www.toolfk.com/zh-cn/tools/base64-to-audio.html base64转音频
/* byte[] decode = Base64.decode(ttsResultDTO.getAudio());
FileUtil.writeBytes(decode, "F:\\tmp\\audio1\\tts" + i + ".wav");*/
}
}
@ -54,4 +91,75 @@ class SpeechDemoServiceApplicationTests {
ChatResDTO chat = difyApiUtil.chat(difyChatReqVO);
System.out.println(JSONUtil.toJsonStr(chat));
}
@Autowired
WebClient webClient;
@Value("${dify.url}")
private String difyUrl;
@Value("${dify.app-auth}")
private String difyAppAuth;
@Test
public void streamChatTest() {
DifyChatReqVO difyChatReqVO = new DifyChatReqVO();
difyChatReqVO.setUser("admin");
DIFYChatReqInputVO inputs = new DIFYChatReqInputVO();
difyChatReqVO.setQuery("我叫什么？");
//difyChatReqVO.setConversation_id("123");
difyChatReqVO.setInputs(inputs);
StringBuilder fullAskString = new StringBuilder();
List<String> audioList = new ArrayList<>();
StringBuilder sentence = new StringBuilder();
webClient.post()
.uri(difyUrl)
.headers(httpHeaders -> {
httpHeaders.setContentType(MediaType.APPLICATION_JSON);
httpHeaders.setBearerAuth(difyAppAuth);
})
.bodyValue(JSON.toJSONString(difyChatReqVO))
.retrieve()
.bodyToFlux(StreamResponse.class)
.map(response -> {
log.info("response: {}", response);
Map<String, String> map = new HashMap<>();
map.put("event", response.getEvent());
if (response.getEvent().equals("message") && response.getAnswer() != null) {
//遍历answer中的每一个字符，判断是否为标点符号，如果是，说明是句子的结尾，将标点符号前的文本拼接到sentence中，并打印，然后清空sentence，如果标点符号后还有文本，将文本拼接到sentence中
for (char ch : response.getAnswer().toCharArray()) {
sentence.append(ch);
fullAskString.append(ch);
if (ch == '。' || ch == '！' || ch == '？' || ch == '，' || ch == '、' || ch == '‘' || ch == '’' || ch == '“' || ch == '”') { // Check for punctuation marks
log.info(sentence.toString());
TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(sentence.toString());
String voiceBaseId = UUID.randomUUID().toString();
audioList.add(voiceBaseId);
map.put("audioId", voiceBaseId);
sentence.setLength(0);
}
}
}
if (response.getEvent().equals("message_end")) {
map.put("sessionId", response.getConversation_id());
if (!sentence.isEmpty()) {
log.info(sentence.toString());
TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(sentence.toString());
String voiceBaseId = UUID.randomUUID().toString();
audioList.add(voiceBaseId);
map.put("audioId", voiceBaseId);
}
}
return null;
}).filter(Objects::nonNull);
while (true){
log.info("sleep....");
ThreadUtil.sleep(1000);
}
}
}

Write Preview
Loading…
Cancel
Save