diff --git a/src/main/java/com/supervision/SpeechDemoServiceApplication.java b/src/main/java/com/supervision/SpeechDemoServiceApplication.java index c929887..d188632 100644 --- a/src/main/java/com/supervision/SpeechDemoServiceApplication.java +++ b/src/main/java/com/supervision/SpeechDemoServiceApplication.java @@ -1,5 +1,6 @@ package com.supervision; +import com.supervision.util.PredefinedAnswerAssistant; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; @@ -11,6 +12,9 @@ public class SpeechDemoServiceApplication { public static void main(String[] args) { SpringApplication.run(SpeechDemoServiceApplication.class, args); + // 加载预定义回答 + PredefinedAnswerAssistant assistant = new PredefinedAnswerAssistant(); + assistant.loadAnswer(); } @Bean public RestTemplate restTemplate() { diff --git a/src/main/java/com/supervision/service/impl/ChatServiceImpl.java b/src/main/java/com/supervision/service/impl/ChatServiceImpl.java index 4877c75..d2b159e 100644 --- a/src/main/java/com/supervision/service/impl/ChatServiceImpl.java +++ b/src/main/java/com/supervision/service/impl/ChatServiceImpl.java @@ -2,9 +2,12 @@ package com.supervision.service.impl; import cn.hutool.core.codec.Base64; import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.date.TimeInterval; import cn.hutool.core.lang.Assert; import cn.hutool.core.util.StrUtil; import cn.hutool.crypto.digest.MD5; +import cn.hutool.json.JSONUtil; import com.alibaba.fastjson.JSON; import com.supervision.dto.dify.ChatResDTO; import com.supervision.dto.paddlespeech.res.TtsResultDTO; @@ -16,10 +19,7 @@ import com.supervision.model.dify.DIFYChatReqInputVO; import com.supervision.model.dify.DifyChatReqVO; import com.supervision.model.dify.StreamResponse; import com.supervision.service.IChatService; -import com.supervision.util.AsrUtil; -import com.supervision.util.DifyApiUtil; -import com.supervision.util.TtsUtil; -import com.supervision.util.WavUtil; +import com.supervision.util.*; import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -36,8 +36,10 @@ import org.springframework.util.StopWatch; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; import java.io.IOException; +import java.time.Duration; import java.util.*; import java.util.HashMap; import java.util.Map; @@ -54,9 +56,12 @@ public class ChatServiceImpl implements IChatService { @Value("${dify.url}") private String difyUrl; + @Value("${dify.app-auth}") private String difyAppAuth; + private final WebClient webClient; + private final DifyApiUtil difyApiUtil; @@ -83,7 +88,12 @@ public class ChatServiceImpl implements IChatService { StringBuilder sentence = new StringBuilder(); log.info("query:{}", query); - return webClient.post() + PredefinedAnswerAssistant predefinedAnswerAssistant = new PredefinedAnswerAssistant(); + TimeInterval timeInterval = DateUtil.timer(); + timeInterval.start(); + timeInterval.start("start"); + log.info("request:时间：{}", DateUtil.now()); + Flux flux = webClient.post() .uri(difyUrl) .headers(httpHeaders -> { httpHeaders.setContentType(MediaType.APPLICATION_JSON); @@ -91,28 +101,41 @@ public class ChatServiceImpl implements IChatService { }) .bodyValue(JSON.toJSONString(difyChatReqVO)) .retrieve() - .bodyToFlux(StreamResponse.class) - .mapNotNull(response -> { + .bodyToFlux(StreamResponse.class); + + + Flux timeoutMono1 = Flux.interval(Duration.ofMillis(1000)).take(8).map(aLong -> { + StreamResponse timeoutResponse = new StreamResponse(); + timeoutResponse.setEvent("timeout"); + return timeoutResponse; + }); + Flux mergedFlux = flux.mergeWith(timeoutMono1); + return mergedFlux.mapNotNull(response -> { + log.info("response:时间：{},响应回答：{}", timeInterval.intervalMs(), JSONUtil.toJsonStr(response)); Map map = new HashMap<>(); map.put("event", response.getEvent()); if (response.getEvent().equals("message") && response.getAnswer() != null) { + log.info("response:时间：{}", DateUtil.now()); + log.info("response:{}", response); //遍历answer中的每一个字符，判断是否为标点符号，如果是，说明是句子的结尾，将标点符号前的文本拼接到sentence中，并打印，然后清空sentence，如果标点符号后还有文本，将文本拼接到sentence中 for (char ch : response.getAnswer().toCharArray()) { sentence.append(ch); fullAskString.append(ch); if (ch == '。' || ch == '！' || ch == '？' || ch == '，' || ch == '、' || ch == '‘' || ch == '’' || ch == '“' || ch == '”') { // Check for punctuation marks - log.info(sentence.toString()); + log.info("响应回答:{}", sentence); TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(sentence.toString()); String voiceBaseId = UUID.randomUUID().toString(); audioList.add(voiceBaseId); audioCache.put(voiceBaseId, ttsResultDTO.getAudio()); map.put("audioId", voiceBaseId); sentence.setLength(0); + predefinedAnswerAssistant.closePreviewAnswer(); return ServerSentEvent.builder(map).build(); } } } if (response.getEvent().equals("message_end")) { + predefinedAnswerAssistant.closePreviewAnswer(); map.put("sessionId", response.getConversation_id()); if (!sentence.isEmpty()) { log.info(sentence.toString()); @@ -126,7 +149,7 @@ public class ChatServiceImpl implements IChatService { String uuid = cn.hutool.core.lang.UUID.randomUUID().toString(); List collect = audioList.stream().map(audioCache::get).filter(Objects::nonNull).collect(Collectors.toList()); String fullAnswer = WavUtil.mergeWavFilesToBase64(collect); - audioCache.put(uuid,fullAnswer); + audioCache.put(uuid, fullAnswer); builder.answerInfo(AnswerInfo.builder().contentType(2).message(fullAskString.toString()).voiceBaseId(uuid).build()); builder.sessionId(response.getConversation_id()); @@ -134,6 +157,29 @@ public class ChatServiceImpl implements IChatService { this.appendDialogCache(response.getConversation_id(), builder.build()); return ServerSentEvent.builder(map).build(); } + if (predefinedAnswerAssistant.overThreshold()) { + if (StrUtil.isEmpty(difyChatReqVO.getConversation_id()) && !predefinedAnswerAssistant.isSkipFirst()) { + predefinedAnswerAssistant.skipFirst(); + String text = predefinedAnswerAssistant.getHelloAnswer(); + predefinedAnswerAssistant.resetInitTime(text); + log.info("超过阈值，返回预定义打招呼回答:{}", text); + String voiceBaseId = MD5.create().digestHex(text); + map.put("audioId", voiceBaseId); + log.info("response前置回答id：{}", voiceBaseId); + audioCache.put(voiceBaseId, predefinedAnswerAssistant.getHelloAnswerAudio(text)); + return ServerSentEvent.builder(map).build(); + } else { + String text = predefinedAnswerAssistant.getPredefinedAnswer(); + log.info("超过阈值，返回预定义回答:{}", text); + predefinedAnswerAssistant.resetInitTime(text); + String voiceBaseId = MD5.create().digestHex(text); + map.put("audioId", voiceBaseId); + log.info("response前置回答id：{}", voiceBaseId); + audioCache.put(voiceBaseId, predefinedAnswerAssistant.getPredefinedAnswerAudio(text)); + return ServerSentEvent.builder(map).build(); + } + + } return null; }).filter(Objects::nonNull); } @@ -186,7 +232,7 @@ public class ChatServiceImpl implements IChatService { public void getAudio(HttpServletResponse response, String audioId) throws IOException { log.info("audioId:{}", audioId); - if (StrUtil.isEmpty(audioId) && StrUtil.equals(audioId, "undefined")) { + if (StrUtil.isEmpty(audioId) || StrUtil.equals(audioId, "undefined")) { return; } Base64.decodeToStream(audioCache.get(audioId), response.getOutputStream(), false); @@ -262,6 +308,8 @@ public class ChatServiceImpl implements IChatService { return sb.toString(); } + + // 示例测试 public static void main(String[] args) { String sampleText = "欢迎来到孟河小镇，体验独特的小镇风情；另外，还有梦和小镇等待你探访。"; diff --git a/src/main/java/com/supervision/util/DifyApiUtil.java b/src/main/java/com/supervision/util/DifyApiUtil.java index 20f8141..83d3b82 100644 --- a/src/main/java/com/supervision/util/DifyApiUtil.java +++ b/src/main/java/com/supervision/util/DifyApiUtil.java @@ -1,5 +1,6 @@ package com.supervision.util; +import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import com.alibaba.fastjson.JSON; import com.supervision.dto.dify.ChatResDTO; @@ -28,6 +29,8 @@ import reactor.core.publisher.Flux; import java.nio.charset.StandardCharsets; +import static com.supervision.common.constant.DifyConstants.CHAT_RESPONSE_MODE_BLOCKING; + @Component @Slf4j @Service @@ -73,8 +76,10 @@ public class DifyApiUtil { ChatResDTO execute; try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpPost httpPost = new HttpPost(difyUrl); - httpPost.setHeader(HttpHeaders.AUTHORIZATION, difyAppAuth); + String authorization = StrUtil.startWith("Bearer ", difyAppAuth) ? difyAppAuth : "Bearer " + difyAppAuth; + httpPost.setHeader(HttpHeaders.AUTHORIZATION, authorization); httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); + difyChatReqVO.setResponse_mode(CHAT_RESPONSE_MODE_BLOCKING); log.info("发起对话：{}", difyChatReqVO); StringEntity entity = new StringEntity(com.alibaba.fastjson.JSONObject.toJSON(difyChatReqVO).toString(), StandardCharsets.UTF_8); httpPost.setEntity(entity); diff --git a/src/main/java/com/supervision/util/PredefinedAnswerAssistant.java b/src/main/java/com/supervision/util/PredefinedAnswerAssistant.java new file mode 100644 index 0000000..7bb5545 --- /dev/null +++ b/src/main/java/com/supervision/util/PredefinedAnswerAssistant.java @@ -0,0 +1,151 @@ +package com.supervision.util; + +import cn.hutool.core.date.DateField; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUnit; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.NumberUtil; +import cn.hutool.core.util.RandomUtil; +import cn.hutool.core.util.StrUtil; +import com.supervision.dto.paddlespeech.res.TtsResultDTO; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; + +/** + * 预定义答案助手 + */ +@Slf4j +public class PredefinedAnswerAssistant { + + /** + * 问候答案 + */ + private static final Map helloAnswerMap = new HashMap<>(); + + /** + * 预定义答案 + */ + private static final Map predefinedAnswerMap = new HashMap<>(); + + /** + * 阈值时间，超过这个阈值就开始调换出一个预置回答 单位毫秒 默认1000毫秒 + */ + private int thresholdMillis = 500; + + /** + * 初始化时间 + */ + private DateTime initTime = DateUtil.date().offset(DateField.MILLISECOND, thresholdMillis); + + private boolean skip = false; + + private boolean skipFirst = false; + + + /** + * 当前时间是否超过阈值 + * @return true 超过阈值 false 未超过阈值 + */ + public boolean overThreshold(){ + + return !skip && initTime.before(new Date()); + } + + public void skipFirst(){ + this.skipFirst = true; + } + + public boolean isSkipFirst(){ + return skipFirst; + } + + /** + * 获取问候答案 + * @return 问候答案 + */ + public String getHelloAnswer() { + List collect = helloAnswerMap.keySet().stream().toList(); + return collect.get(RandomUtil.randomInt(0, collect.size() - 1)); + } + + public void setThresholdMillis(int thresholdMillis) { + this.thresholdMillis = thresholdMillis; + } + + public void closePreviewAnswer(){ + skip = true; + } + + /** + * 获取问候答案音频 + * @param text 问候答案 + * @return 问候答案音频 + */ + public String getHelloAnswerAudio(String text) { + return helloAnswerMap.get(text).getAudio(); + } + + + + /** + * 获取预定义答案 + * @return 预定义答案 + */ + public String getPredefinedAnswer() { + List collect = predefinedAnswerMap.keySet().stream().toList(); + return collect.get(RandomUtil.randomInt(0, collect.size() - 1)); + } + + public String getPredefinedAnswerAudio(String text) { + return predefinedAnswerMap.get(text).getAudio(); + } + + public void resetInitTime(String text){ + if (StrUtil.isEmpty(text)){ + return; + } + TtsResultDTO ttsResultDTO = null == predefinedAnswerMap.get(text) ? helloAnswerMap.get(text) : predefinedAnswerMap.get(text); + if (null == ttsResultDTO){ + return; + } + double v = NumberUtil.parseDouble(ttsResultDTO.getDuration()) * 1000; + this.initTime = DateUtil.offsetMillisecond(DateUtil.date(), NumberUtil.parseInt(String.valueOf(v))); + } + + + /** + * 加载问候答案 + */ + public void loadAnswer() { + + List helloAnswerList = List.of("您好，我是您的康养顾问小苏，很开心接到您的来电","您好，我是您的康养顾问小苏，感谢您的来电", + "您好，我是您的康养顾问小陈，很高兴接到您的电话"); + + log.info("开始加载问候答案...."); + for (String text : helloAnswerList) { + if (helloAnswerMap.containsKey(text)) { + continue; + } + TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(text); + helloAnswerMap.put(text,ttsResultDTO); + } + log.info("问候答案加载完成...."); + + List predefinedAnswerList = List.of("您好，我是您的康养顾问小陈，很高兴接到您的电话", "我会结合最新的资料，为您提供最贴心的解答", + "结合最新的信息，为您提供最合适的建议", "结合最新的资料为您提供详细的解答", "根据掌握的高黎贡勐赫小镇信息，为您提供最准确的服务与建议", + "根据最新动态，为您提供最精准的解答与建议", "结合最新情况，为您提供最贴心的服务与支持"); + + log.info("开始加载预定义答案...."); + for (String text : predefinedAnswerList) { + if (predefinedAnswerMap.containsKey(text)){ + continue; + } + TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(text); + predefinedAnswerMap.put(text,ttsResultDTO); + } + log.info("预定义答案加载完成...."); + } + + +} diff --git a/src/main/java/com/supervision/util/TtsUtil.java b/src/main/java/com/supervision/util/TtsUtil.java index 92ab149..014733d 100644 --- a/src/main/java/com/supervision/util/TtsUtil.java +++ b/src/main/java/com/supervision/util/TtsUtil.java @@ -31,6 +31,5 @@ public class TtsUtil { } catch (Exception e) { throw new BusinessException("语音转换文字失败", e); } - } } diff --git a/src/test/java/com/supervision/SpeechDemoServiceApplicationTests.java b/src/test/java/com/supervision/SpeechDemoServiceApplicationTests.java index c0fad56..82e48fa 100644 --- a/src/test/java/com/supervision/SpeechDemoServiceApplicationTests.java +++ b/src/test/java/com/supervision/SpeechDemoServiceApplicationTests.java @@ -1,18 +1,35 @@ package com.supervision; +import cn.hutool.core.codec.Base64; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.thread.ThreadUtil; import cn.hutool.json.JSONUtil; +import com.alibaba.fastjson.JSON; import com.supervision.dto.dify.ChatResDTO; import com.supervision.dto.paddlespeech.res.TtsResultDTO; +import com.supervision.dto.robot.AnswerInfo; +import com.supervision.dto.robot.AskInfo; import com.supervision.model.dify.DIFYChatReqInputVO; import com.supervision.model.dify.DifyChatReqVO; +import com.supervision.model.dify.StreamResponse; import com.supervision.util.AsrUtil; import com.supervision.util.DifyApiUtil; import com.supervision.util.TtsUtil; +import com.supervision.util.WavUtil; import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.http.codec.ServerSentEvent; +import org.springframework.web.reactive.function.client.WebClient; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j @SpringBootTest class SpeechDemoServiceApplicationTests { @@ -23,9 +40,29 @@ class SpeechDemoServiceApplicationTests { @Test void testTtsTransform() { - TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform("欢迎来电，我是您的康养顾问小苏。关于您的问题，我们已经为您查询到了相关信息。"); - System.out.println(JSONUtil.toJsonStr(ttsResultDTO)); - // https://www.toolfk.com/zh-cn/tools/base64-to-audio.html base64转音频 + + String text = """ + 您好，我是您的康养顾问小苏，很开心接到您的来电 + 您好，我是您的康养顾问小苏，感谢您的来电 + 您好，我是您的康养顾问小陈，很高兴接到您的电话 + 关于您的问题，我已经为您整理好了相关信息 + 我会结合最新的资料，为您提供最贴心的解答 + 结合最新的信息，为您提供最合适的建议 + 结合最新的资料为您提供详细的解答 + 根据掌握的高黎贡勐赫小镇信息，为您提供最准确的服务与建议 + 根据最新动态，为您提供最精准的解答与建议 + 结合最新情况，为您提供最贴心的服务与支持 + """; + + List collect = Arrays.stream(text.split("\n")).filter(s -> !s.isBlank()).toList(); + for (int i = 0; i < collect.size(); i++) { + TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(collect.get(i)); + String jsonStr = JSONUtil.toJsonStr(ttsResultDTO); + System.out.println(jsonStr); + // https://www.toolfk.com/zh-cn/tools/base64-to-audio.html base64转音频 + /* byte[] decode = Base64.decode(ttsResultDTO.getAudio()); + FileUtil.writeBytes(decode, "F:\\tmp\\audio1\\tts" + i + ".wav");*/ + } } @@ -54,4 +91,75 @@ class SpeechDemoServiceApplicationTests { ChatResDTO chat = difyApiUtil.chat(difyChatReqVO); System.out.println(JSONUtil.toJsonStr(chat)); } + + @Autowired + WebClient webClient; + @Value("${dify.url}") + private String difyUrl; + + @Value("${dify.app-auth}") + private String difyAppAuth; + @Test + public void streamChatTest() { + + DifyChatReqVO difyChatReqVO = new DifyChatReqVO(); + difyChatReqVO.setUser("admin"); + DIFYChatReqInputVO inputs = new DIFYChatReqInputVO(); + difyChatReqVO.setQuery("我叫什么？"); + //difyChatReqVO.setConversation_id("123"); + difyChatReqVO.setInputs(inputs); + + StringBuilder fullAskString = new StringBuilder(); + List audioList = new ArrayList<>(); + StringBuilder sentence = new StringBuilder(); + + webClient.post() + .uri(difyUrl) + .headers(httpHeaders -> { + httpHeaders.setContentType(MediaType.APPLICATION_JSON); + httpHeaders.setBearerAuth(difyAppAuth); + }) + .bodyValue(JSON.toJSONString(difyChatReqVO)) + .retrieve() + .bodyToFlux(StreamResponse.class) + .map(response -> { + + log.info("response: {}", response); + + Map map = new HashMap<>(); + map.put("event", response.getEvent()); + if (response.getEvent().equals("message") && response.getAnswer() != null) { + //遍历answer中的每一个字符，判断是否为标点符号，如果是，说明是句子的结尾，将标点符号前的文本拼接到sentence中，并打印，然后清空sentence，如果标点符号后还有文本，将文本拼接到sentence中 + for (char ch : response.getAnswer().toCharArray()) { + sentence.append(ch); + fullAskString.append(ch); + if (ch == '。' || ch == '！' || ch == '？' || ch == '，' || ch == '、' || ch == '‘' || ch == '’' || ch == '“' || ch == '”') { // Check for punctuation marks + log.info(sentence.toString()); + TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(sentence.toString()); + String voiceBaseId = UUID.randomUUID().toString(); + audioList.add(voiceBaseId); + map.put("audioId", voiceBaseId); + sentence.setLength(0); + } + } + } + if (response.getEvent().equals("message_end")) { + map.put("sessionId", response.getConversation_id()); + if (!sentence.isEmpty()) { + log.info(sentence.toString()); + TtsResultDTO ttsResultDTO = TtsUtil.ttsTransform(sentence.toString()); + String voiceBaseId = UUID.randomUUID().toString(); + audioList.add(voiceBaseId); + map.put("audioId", voiceBaseId); + } + } + return null; + }).filter(Objects::nonNull); + + + while (true){ + log.info("sleep...."); + ThreadUtil.sleep(1000); + } + } }