From acf425690446298d0d319479af1c4de96841c296 Mon Sep 17 00:00:00 2001 From: liu Date: Thu, 26 Oct 2023 14:07:01 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96,=E6=8F=90=E5=8F=96=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../supervision/controller/AskController.java | 6 +- .../com/supervision/paddlespeech/AsrUtil.java | 44 ++++++ .../com/supervision/paddlespeech/TtsUtil.java | 40 ++++++ .../dto}/req/AsrReqDTO.java | 2 +- .../dto}/req/TtsReqDTO.java | 2 +- .../dto}/res/AsrResultDTO.java | 2 +- .../dto}/res/PaddleSpeechResDTO.java | 2 +- .../dto}/res/TtsResultDTO.java | 2 +- .../pojo/rasa/train/intent/NluTemplate.java | 11 -- .../java/com/supervision/rasa/RasaUtil.java | 27 ++++ .../{pojo/rasa => rasa/dto}/RasaReqDTO.java | 2 +- .../{pojo/rasa => rasa/dto}/RasaResDTO.java | 2 +- .../dto/train}/DomainYmlTemplate.java | 2 +- .../intent => rasa/dto/train}/Intent.java | 2 +- .../dto/train}/NluYmlTemplate.java | 2 +- .../dto/train}/RuleYmlTemplate.java | 2 +- .../com/supervision/service/AskService.java | 7 +- .../service/impl/AskServiceImpl.java | 98 +++++--------- .../service/impl/RasaServiceImpl.java | 8 +- .../com/supervision/util/SpringBeanUtil.java | 127 ++++++++++++++++++ .../supervision/websocket/dto/ActionDTO.java | 14 ++ .../websocket/dto/SocketMessageDTO.java | 26 +++- .../handler/AskWebSocketHandler.java | 10 +- 23 files changed, 339 insertions(+), 101 deletions(-) create mode 100644 virtual-patient-web/src/main/java/com/supervision/paddlespeech/AsrUtil.java create mode 100644 virtual-patient-web/src/main/java/com/supervision/paddlespeech/TtsUtil.java rename virtual-patient-web/src/main/java/com/supervision/{pojo/paddlespeech => paddlespeech/dto}/req/AsrReqDTO.java (87%) rename virtual-patient-web/src/main/java/com/supervision/{pojo/paddlespeech => paddlespeech/dto}/req/TtsReqDTO.java (90%) rename virtual-patient-web/src/main/java/com/supervision/{pojo/paddlespeech => paddlespeech/dto}/res/AsrResultDTO.java (66%) rename virtual-patient-web/src/main/java/com/supervision/{pojo/paddlespeech => paddlespeech/dto}/res/PaddleSpeechResDTO.java (78%) rename virtual-patient-web/src/main/java/com/supervision/{pojo/paddlespeech => paddlespeech/dto}/res/TtsResultDTO.java (85%) delete mode 100644 virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/NluTemplate.java create mode 100644 virtual-patient-web/src/main/java/com/supervision/rasa/RasaUtil.java rename virtual-patient-web/src/main/java/com/supervision/{pojo/rasa => rasa/dto}/RasaReqDTO.java (76%) rename virtual-patient-web/src/main/java/com/supervision/{pojo/rasa => rasa/dto}/RasaResDTO.java (76%) rename virtual-patient-web/src/main/java/com/supervision/{pojo/rasa/train/intent => rasa/dto/train}/DomainYmlTemplate.java (91%) rename virtual-patient-web/src/main/java/com/supervision/{pojo/rasa/train/intent => rasa/dto/train}/Intent.java (82%) rename virtual-patient-web/src/main/java/com/supervision/{pojo/rasa/train/intent => rasa/dto/train}/NluYmlTemplate.java (87%) rename virtual-patient-web/src/main/java/com/supervision/{pojo/rasa/train/intent => rasa/dto/train}/RuleYmlTemplate.java (93%) create mode 100644 virtual-patient-web/src/main/java/com/supervision/util/SpringBeanUtil.java create mode 100644 virtual-patient-web/src/main/java/com/supervision/websocket/dto/ActionDTO.java diff --git a/virtual-patient-web/src/main/java/com/supervision/controller/AskController.java b/virtual-patient-web/src/main/java/com/supervision/controller/AskController.java index 93b4ea89..f07ccff1 100644 --- a/virtual-patient-web/src/main/java/com/supervision/controller/AskController.java +++ b/virtual-patient-web/src/main/java/com/supervision/controller/AskController.java @@ -31,7 +31,7 @@ public class AskController { session.sendMessage(new TextMessage(message)); } - @ApiOperation("接收页面的语音消息") + @ApiOperation("接收页面的语音消息(暂时不用这个接口,主要使用websocket进行通信)") @PostMapping("/receiveVoiceFile") public String receiveVoiceFile(@RequestParam("file") MultipartFile file) throws IOException { return askService.receiveVoiceFile(file); @@ -45,8 +45,8 @@ public class AskController { @ApiOperation("进行对话") @GetMapping("conversation") - public List conversation(String question) { - return askService.conversation(question); + public List conversation(String question, String sessionId) { + return askService.conversation(question, sessionId); } @ApiOperation("查询对话历史") diff --git a/virtual-patient-web/src/main/java/com/supervision/paddlespeech/AsrUtil.java b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/AsrUtil.java new file mode 100644 index 00000000..bbaa4253 --- /dev/null +++ b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/AsrUtil.java @@ -0,0 +1,44 @@ +package com.supervision.paddlespeech; + +import cn.hutool.core.codec.Base64; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.http.HttpUtil; +import cn.hutool.json.JSONUtil; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.supervision.exception.BusinessException; +import com.supervision.paddlespeech.dto.req.AsrReqDTO; +import com.supervision.paddlespeech.dto.res.AsrResultDTO; +import com.supervision.paddlespeech.dto.res.PaddleSpeechResDTO; +import com.supervision.util.SpringBeanUtil; +import org.springframework.core.env.Environment; + +public class AsrUtil { + + private static final String ASR_URL = SpringBeanUtil.getBean(Environment.class).getProperty("paddle-speech.asr"); + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public static String asrTransformByBytes(byte[] bytes) { + // 首先编码为base64编码 + String encode = Base64.encode(bytes); + return asrTransformByBytes(encode); + } + + public static String asrTransformByBytes(String voiceBase64){ + // 这里调用Python的接口,将文字转换为语音 + String post = HttpUtil.post(ASR_URL, JSONUtil.toJsonStr(new AsrReqDTO(voiceBase64))); + try { + PaddleSpeechResDTO response = objectMapper.readValue(post, new TypeReference>() { + }); + if (!response.getSuccess() || ObjectUtil.isEmpty(response.getResult())) { + throw new BusinessException("语音转换文字失败"); + } + return response.getResult().getTranscription(); + + } catch (Exception e) { + throw new BusinessException("语音转换文字失败"); + } + + } +} diff --git a/virtual-patient-web/src/main/java/com/supervision/paddlespeech/TtsUtil.java b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/TtsUtil.java new file mode 100644 index 00000000..9d307f5f --- /dev/null +++ b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/TtsUtil.java @@ -0,0 +1,40 @@ +package com.supervision.paddlespeech; + +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.http.HttpUtil; +import cn.hutool.json.JSONUtil; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.supervision.exception.BusinessException; +import com.supervision.paddlespeech.dto.req.TtsReqDTO; +import com.supervision.paddlespeech.dto.res.PaddleSpeechResDTO; +import com.supervision.paddlespeech.dto.res.TtsResultDTO; +import com.supervision.pojo.vo.ReplyVoiceResVO; +import com.supervision.util.SpringBeanUtil; +import org.springframework.core.env.Environment; + +public class TtsUtil { + + private static final String TTS_URL = SpringBeanUtil.getBean(Environment.class).getProperty("paddle-speech.tts"); + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public static ReplyVoiceResVO ttsTransform(String str) { + // 构建 + String post = HttpUtil.post(TTS_URL, JSONUtil.toJsonStr(new TtsReqDTO(str))); + try { + PaddleSpeechResDTO response = objectMapper.readValue(post, new TypeReference>() { + }); + if (!response.getSuccess() || ObjectUtil.isEmpty(response.getResult())) { + throw new BusinessException("文字转换语音失败"); + } + ReplyVoiceResVO resVO = new ReplyVoiceResVO(); + resVO.setVoice(response.getResult().getAudio()); + resVO.setText(str); + return resVO; + } catch (Exception e) { + throw new BusinessException("语音转换文字失败"); + } + + } +} diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/req/AsrReqDTO.java b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/req/AsrReqDTO.java similarity index 87% rename from virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/req/AsrReqDTO.java rename to virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/req/AsrReqDTO.java index a5d49703..902cc710 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/req/AsrReqDTO.java +++ b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/req/AsrReqDTO.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.paddlespeech.req; +package com.supervision.paddlespeech.dto.req; import lombok.Data; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/req/TtsReqDTO.java b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/req/TtsReqDTO.java similarity index 90% rename from virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/req/TtsReqDTO.java rename to virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/req/TtsReqDTO.java index aac30cc4..d3f96e55 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/req/TtsReqDTO.java +++ b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/req/TtsReqDTO.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.paddlespeech.req; +package com.supervision.paddlespeech.dto.req; import lombok.Data; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/res/AsrResultDTO.java b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/res/AsrResultDTO.java similarity index 66% rename from virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/res/AsrResultDTO.java rename to virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/res/AsrResultDTO.java index d1803872..85563f4b 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/res/AsrResultDTO.java +++ b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/res/AsrResultDTO.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.paddlespeech.res; +package com.supervision.paddlespeech.dto.res; import lombok.Data; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/res/PaddleSpeechResDTO.java b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/res/PaddleSpeechResDTO.java similarity index 78% rename from virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/res/PaddleSpeechResDTO.java rename to virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/res/PaddleSpeechResDTO.java index 8b17ffb8..bca8f303 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/res/PaddleSpeechResDTO.java +++ b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/res/PaddleSpeechResDTO.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.paddlespeech.res; +package com.supervision.paddlespeech.dto.res; import lombok.Data; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/res/TtsResultDTO.java b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/res/TtsResultDTO.java similarity index 85% rename from virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/res/TtsResultDTO.java rename to virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/res/TtsResultDTO.java index 07d75b0a..f6f7c22c 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/paddlespeech/res/TtsResultDTO.java +++ b/virtual-patient-web/src/main/java/com/supervision/paddlespeech/dto/res/TtsResultDTO.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.paddlespeech.res; +package com.supervision.paddlespeech.dto.res; import lombok.Data; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/NluTemplate.java b/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/NluTemplate.java deleted file mode 100644 index c0e45a5d..00000000 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/NluTemplate.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.supervision.pojo.rasa.train.intent; - -import lombok.Data; - -import java.util.List; - -@Data -public class NluTemplate { - - -} diff --git a/virtual-patient-web/src/main/java/com/supervision/rasa/RasaUtil.java b/virtual-patient-web/src/main/java/com/supervision/rasa/RasaUtil.java new file mode 100644 index 00000000..bd653df6 --- /dev/null +++ b/virtual-patient-web/src/main/java/com/supervision/rasa/RasaUtil.java @@ -0,0 +1,27 @@ +package com.supervision.rasa; + +import cn.hutool.http.HttpUtil; +import cn.hutool.json.JSONUtil; +import com.supervision.rasa.dto.RasaReqDTO; +import com.supervision.rasa.dto.RasaResDTO; +import com.supervision.util.SpringBeanUtil; +import org.springframework.core.env.Environment; + +import java.util.List; +import java.util.stream.Collectors; + +public class RasaUtil { + + private static final String RASA_URL = SpringBeanUtil.getBean(Environment.class).getProperty("rasa.url"); + + public static List talkRasa(String question, String sessionId) { + RasaReqDTO rasaReqDTO = new RasaReqDTO(); + rasaReqDTO.setSender(sessionId); + rasaReqDTO.setMessage(question); + String post = HttpUtil.post(RASA_URL, JSONUtil.toJsonStr(rasaReqDTO)); + List list = JSONUtil.toList(post, RasaResDTO.class); + return list.stream().map(RasaResDTO::getText).collect(Collectors.toList()); + } + + +} diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/RasaReqDTO.java b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/RasaReqDTO.java similarity index 76% rename from virtual-patient-web/src/main/java/com/supervision/pojo/rasa/RasaReqDTO.java rename to virtual-patient-web/src/main/java/com/supervision/rasa/dto/RasaReqDTO.java index 0634dce5..1daef8d9 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/RasaReqDTO.java +++ b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/RasaReqDTO.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.rasa; +package com.supervision.rasa.dto; import lombok.Data; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/RasaResDTO.java b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/RasaResDTO.java similarity index 76% rename from virtual-patient-web/src/main/java/com/supervision/pojo/rasa/RasaResDTO.java rename to virtual-patient-web/src/main/java/com/supervision/rasa/dto/RasaResDTO.java index 5b73187e..5f1042e8 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/RasaResDTO.java +++ b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/RasaResDTO.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.rasa; +package com.supervision.rasa.dto; import lombok.Data; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/DomainYmlTemplate.java b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/DomainYmlTemplate.java similarity index 91% rename from virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/DomainYmlTemplate.java rename to virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/DomainYmlTemplate.java index 2c1f381c..418164b3 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/DomainYmlTemplate.java +++ b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/DomainYmlTemplate.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.rasa.train.intent; +package com.supervision.rasa.dto.train; import lombok.Data; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/Intent.java b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/Intent.java similarity index 82% rename from virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/Intent.java rename to virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/Intent.java index eb7bd9a8..74dcfd9d 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/Intent.java +++ b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/Intent.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.rasa.train.intent; +package com.supervision.rasa.dto.train; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/NluYmlTemplate.java b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/NluYmlTemplate.java similarity index 87% rename from virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/NluYmlTemplate.java rename to virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/NluYmlTemplate.java index e4eb283c..b2a7f689 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/NluYmlTemplate.java +++ b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/NluYmlTemplate.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.rasa.train.intent; +package com.supervision.rasa.dto.train; import lombok.Data; import org.yaml.snakeyaml.nodes.Tag; diff --git a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/RuleYmlTemplate.java b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/RuleYmlTemplate.java similarity index 93% rename from virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/RuleYmlTemplate.java rename to virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/RuleYmlTemplate.java index 47572cb2..a690fdca 100644 --- a/virtual-patient-web/src/main/java/com/supervision/pojo/rasa/train/intent/RuleYmlTemplate.java +++ b/virtual-patient-web/src/main/java/com/supervision/rasa/dto/train/RuleYmlTemplate.java @@ -1,4 +1,4 @@ -package com.supervision.pojo.rasa.train.intent; +package com.supervision.rasa.dto.train; import cn.hutool.core.map.MapUtil; import lombok.AllArgsConstructor; diff --git a/virtual-patient-web/src/main/java/com/supervision/service/AskService.java b/virtual-patient-web/src/main/java/com/supervision/service/AskService.java index b975aae3..c84f30ac 100644 --- a/virtual-patient-web/src/main/java/com/supervision/service/AskService.java +++ b/virtual-patient-web/src/main/java/com/supervision/service/AskService.java @@ -2,6 +2,7 @@ package com.supervision.service; import com.fasterxml.jackson.core.JsonProcessingException; import com.supervision.pojo.vo.ReplyVoiceResVO; +import com.supervision.websocket.dto.SocketMessageDTO; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.multipart.MultipartFile; @@ -10,11 +11,13 @@ import java.util.List; public interface AskService { + void handlerMessageBySocket(SocketMessageDTO socketMessageDTO); + String receiveVoiceFile(MultipartFile file) throws IOException; - ReplyVoiceResVO replyVoice() ; + ReplyVoiceResVO replyVoice(); - List conversation(String question); + List conversation(String question, String sessionId); } diff --git a/virtual-patient-web/src/main/java/com/supervision/service/impl/AskServiceImpl.java b/virtual-patient-web/src/main/java/com/supervision/service/impl/AskServiceImpl.java index 8ad72e22..49b9a328 100644 --- a/virtual-patient-web/src/main/java/com/supervision/service/impl/AskServiceImpl.java +++ b/virtual-patient-web/src/main/java/com/supervision/service/impl/AskServiceImpl.java @@ -1,22 +1,17 @@ package com.supervision.service.impl; -import cn.hutool.core.codec.Base64; -import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.StrUtil; import cn.hutool.http.HttpUtil; import cn.hutool.json.JSONUtil; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.supervision.exception.BusinessException; -import com.supervision.pojo.paddlespeech.req.AsrReqDTO; -import com.supervision.pojo.paddlespeech.req.TtsReqDTO; -import com.supervision.pojo.paddlespeech.res.AsrResultDTO; -import com.supervision.pojo.paddlespeech.res.PaddleSpeechResDTO; -import com.supervision.pojo.paddlespeech.res.TtsResultDTO; -import com.supervision.pojo.rasa.RasaReqDTO; -import com.supervision.pojo.rasa.RasaResDTO; +import com.supervision.paddlespeech.AsrUtil; +import com.supervision.paddlespeech.TtsUtil; +import com.supervision.rasa.RasaUtil; +import com.supervision.rasa.dto.RasaReqDTO; +import com.supervision.rasa.dto.RasaResDTO; import com.supervision.pojo.vo.ReplyVoiceResVO; import com.supervision.service.AskService; +import com.supervision.websocket.dto.SocketMessageDTO; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -24,7 +19,6 @@ import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.util.List; -import java.util.UUID; import java.util.stream.Collectors; @Service @@ -32,73 +26,43 @@ import java.util.stream.Collectors; public class AskServiceImpl implements AskService { - @Value("${paddle-speech.tts}") - private String tts; - - @Value("${paddle-speech.asr}") - private String asr; + @Override + public void handlerMessageBySocket(SocketMessageDTO socketMessageDTO) { + // 首先获取消息的类型 + String text; + if (0 == socketMessageDTO.getMessageType()) { + // 如果是语音消息 + // 调用百度接口,语音转文字 + text = AsrUtil.asrTransformByBytes(socketMessageDTO.getVoiceMessage()); + } else { + text = socketMessageDTO.getTextMessage(); + } + if (StrUtil.isBlank(text)) { + throw new BusinessException("语音消息不能为空"); + } + // 进行rasa对话 - @Value("${rasa.url}") - private String rasa; - private final ObjectMapper objectMapper = new ObjectMapper(); + } @Override public String receiveVoiceFile(MultipartFile file) throws IOException { - return asrTransform(file.getBytes()); + // 获取音频对应的文字 + String askQuestion = AsrUtil.asrTransformByBytes(file.getBytes()); + // 调用rasa获取文字内容 + throw new BusinessException("暂未实现"); } @Override - public ReplyVoiceResVO replyVoice() { + public ReplyVoiceResVO replyVoice() { String text = "测试:这是文字转语音的测试,测试是否OK"; - return ttsTransform(text); + return TtsUtil.ttsTransform(text); } - private String asrTransform(byte[] bytes) { - // 首先编码为base64编码 - String encode = Base64.encode(bytes); - // 这里调用Python的接口,将文字转换为语音 - String post = HttpUtil.post(asr, JSONUtil.toJsonStr(new AsrReqDTO(encode))); - try { - PaddleSpeechResDTO response = objectMapper.readValue(post, new TypeReference>() {}); - if (!response.getSuccess() || ObjectUtil.isEmpty(response.getResult())) { - throw new BusinessException("语音转换文字失败"); - } - return response.getResult().getTranscription(); - - }catch (Exception e){ - throw new BusinessException("语音转换文字失败"); - } - } - - private ReplyVoiceResVO ttsTransform(String str) { - // 构建 - String post = HttpUtil.post(tts, JSONUtil.toJsonStr(new TtsReqDTO(str))); - try { - PaddleSpeechResDTO response = objectMapper.readValue(post, new TypeReference>() { - }); - if (!response.getSuccess() || ObjectUtil.isEmpty(response.getResult())) { - throw new BusinessException("文字转换语音失败"); - } - ReplyVoiceResVO resVO = new ReplyVoiceResVO(); - resVO.setVoice(response.getResult().getAudio()); - resVO.setText(str); - return resVO; - }catch (Exception e){ - throw new BusinessException("语音转换文字失败"); - } - - } @Override - public List conversation(String question) { - RasaReqDTO rasaReqDTO = new RasaReqDTO(); - rasaReqDTO.setSender("tester1"); - rasaReqDTO.setMessage(question); - - String post = HttpUtil.post(rasa, JSONUtil.toJsonStr(rasaReqDTO)); - List list = JSONUtil.toList(post, RasaResDTO.class); - return list.stream().map(RasaResDTO::getText).collect(Collectors.toList()); + public List conversation(String question, String sessionId) { + return RasaUtil.talkRasa(question,sessionId); } } diff --git a/virtual-patient-web/src/main/java/com/supervision/service/impl/RasaServiceImpl.java b/virtual-patient-web/src/main/java/com/supervision/service/impl/RasaServiceImpl.java index 3d9cda84..423a47c8 100644 --- a/virtual-patient-web/src/main/java/com/supervision/service/impl/RasaServiceImpl.java +++ b/virtual-patient-web/src/main/java/com/supervision/service/impl/RasaServiceImpl.java @@ -6,10 +6,10 @@ import cn.hutool.core.io.IoUtil; import cn.hutool.json.JSONUtil; import com.supervision.exception.BusinessException; import com.supervision.model.*; -import com.supervision.pojo.rasa.train.intent.DomainYmlTemplate; -import com.supervision.pojo.rasa.train.intent.Intent; -import com.supervision.pojo.rasa.train.intent.NluYmlTemplate; -import com.supervision.pojo.rasa.train.intent.RuleYmlTemplate; +import com.supervision.rasa.dto.train.DomainYmlTemplate; +import com.supervision.rasa.dto.train.Intent; +import com.supervision.rasa.dto.train.NluYmlTemplate; +import com.supervision.rasa.dto.train.RuleYmlTemplate; import com.supervision.service.*; import freemarker.template.Configuration; import freemarker.template.Template; diff --git a/virtual-patient-web/src/main/java/com/supervision/util/SpringBeanUtil.java b/virtual-patient-web/src/main/java/com/supervision/util/SpringBeanUtil.java new file mode 100644 index 00000000..9140c1ea --- /dev/null +++ b/virtual-patient-web/src/main/java/com/supervision/util/SpringBeanUtil.java @@ -0,0 +1,127 @@ +package com.supervision.util; + +import cn.hutool.core.util.ArrayUtil; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +import java.util.Map; + +/*** + * @author ljt + * @since 2020/8/4 17:37 + * + */ +@Component +@Lazy(false) +public class SpringBeanUtil implements ApplicationContextAware { + + private static ApplicationContext applicationContext; + + /** + * 获取applicationContext + * + * @return ApplicationContext + */ + public static ApplicationContext getApplicationContext() { + return applicationContext; + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) { + SpringBeanUtil.applicationContext = applicationContext; + } + + //通过name获取 Bean. + + /** + * 通过name获取 Bean + * + * @param Bean类型 + * @param name Bean名称 + * @return Bean + */ + @SuppressWarnings("unchecked") + public static T getBean(String name) { + return (T) applicationContext.getBean(name); + } + + /** + * 通过class获取Bean + * + * @param Bean类型 + * @param clazz Bean类 + * @return Bean对象 + */ + public static T getBean(Class clazz) { + return applicationContext.getBean(clazz); + } + + /** + * 通过name,以及Clazz返回指定的Bean + * + * @param bean类型 + * @param name Bean名称 + * @param clazz bean类型 + * @return Bean对象 + */ + public static T getBean(String name, Class clazz) { + return applicationContext.getBean(name, clazz); + } + + /** + * 获取指定类型对应的所有Bean，包括子类 + * + * @param Bean类型 + * @param type 类、接口，null表示获取所有bean + * @return 类型对应的bean，key是bean注册的name，value是Bean + * @since 5.3.3 + */ + public static Map getBeansOfType(Class type) { + return applicationContext.getBeansOfType(type); + } + + /** + * 获取指定类型对应的Bean名称，包括子类 + * + * @param type 类、接口，null表示获取所有bean名称 + * @return bean名称 + * @since 5.3.3 + */ + public static String[] getBeanNamesForType(Class type) { + return applicationContext.getBeanNamesForType(type); + } + + /** + * 获取配置文件配置项的值 + * + * @param key 配置项key + * @return 属性值 + * @since 5.3.3 + */ + public static String getProperty(String key) { + return applicationContext.getEnvironment().getProperty(key); + } + + /** + * 获取当前的环境配置，无配置返回null + * + * @return 当前的环境配置 + * @since 5.3.3 + */ + public static String[] getActiveProfiles() { + return applicationContext.getEnvironment().getActiveProfiles(); + } + + /** + * 获取当前的环境配置，当有多个环境配置时，只获取第一个 + * + * @return 当前的环境配置 + * @since 5.3.3 + */ + public static String getActiveProfile() { + final String[] activeProfiles = getActiveProfiles(); + return ArrayUtil.isNotEmpty(activeProfiles) ? activeProfiles[0] : null; + } +} diff --git a/virtual-patient-web/src/main/java/com/supervision/websocket/dto/ActionDTO.java b/virtual-patient-web/src/main/java/com/supervision/websocket/dto/ActionDTO.java new file mode 100644 index 00000000..babff18d --- /dev/null +++ b/virtual-patient-web/src/main/java/com/supervision/websocket/dto/ActionDTO.java @@ -0,0 +1,14 @@ +package com.supervision.websocket.dto; + +import lombok.Data; + +@Data +public class ActionDTO { + + /** + * type为枚举,分为2中类型,tool 体格检查工具 ancillary 辅助检查 + */ + private String actionType; + + private String actionId; +} diff --git a/virtual-patient-web/src/main/java/com/supervision/websocket/dto/SocketMessageDTO.java b/virtual-patient-web/src/main/java/com/supervision/websocket/dto/SocketMessageDTO.java index c469007f..16c1fd74 100644 --- a/virtual-patient-web/src/main/java/com/supervision/websocket/dto/SocketMessageDTO.java +++ b/virtual-patient-web/src/main/java/com/supervision/websocket/dto/SocketMessageDTO.java @@ -1,13 +1,37 @@ package com.supervision.websocket.dto; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; import lombok.Data; @Data +@ApiModel public class SocketMessageDTO { + @ApiModelProperty("socket链接ID") private String socketId; + @ApiModelProperty("当前用户ID") private String userId; - private String data; + @ApiModelProperty("文字消息内容") + private String textMessage; + + @ApiModelProperty("语音消息内容") + private String voiceMessage; + + @ApiModelProperty("若有动作,这是需要执行的动作内容") + private ActionDTO action; + + /** + * 后端返回给前端时使用,表示该是消息还是action动作,0消息,1动作 + */ + @ApiModelProperty("后端返回给前端时使用,表示该是消息还是action动作,0消息,1动作") + private Integer type; + + /** + * 表示是消息还是action动作,0语音,1文字 + */ + @ApiModelProperty("前端到后端使用,表示是语音还是文字,0语音,1文字") + private Integer messageType; } diff --git a/virtual-patient-web/src/main/java/com/supervision/websocket/handler/AskWebSocketHandler.java b/virtual-patient-web/src/main/java/com/supervision/websocket/handler/AskWebSocketHandler.java index 34c8ddeb..32f93691 100644 --- a/virtual-patient-web/src/main/java/com/supervision/websocket/handler/AskWebSocketHandler.java +++ b/virtual-patient-web/src/main/java/com/supervision/websocket/handler/AskWebSocketHandler.java @@ -1,9 +1,12 @@ package com.supervision.websocket.handler; import cn.hutool.json.JSONUtil; +import com.supervision.service.AskService; +import com.supervision.util.SpringBeanUtil; import com.supervision.websocket.cache.WebSocketUserCache; import com.supervision.websocket.dto.SocketMessageDTO; import lombok.extern.slf4j.Slf4j; +import org.springframework.lang.NonNullApi; import org.springframework.web.socket.CloseStatus; import org.springframework.web.socket.TextMessage; import org.springframework.web.socket.WebSocketSession; @@ -21,14 +24,17 @@ public class AskWebSocketHandler extends TextWebSocketHandler { // 返回sessionId到前端 SocketMessageDTO socketMessageDTO = new SocketMessageDTO(); socketMessageDTO.setSocketId(id); - socketMessageDTO.setData("链接成功"); session.sendMessage(new TextMessage(JSONUtil.toJsonStr(socketMessageDTO))); } @Override protected void handleTextMessage(WebSocketSession session, TextMessage message) { - // 处理接收到的文本消息 + // 处理接收到的消息 log.info("收到消息:{}", message.toString()); + // 这里反序列化消息,将消息形成固定的格式 + SocketMessageDTO socketMessageDTO = JSONUtil.toBean(message.getPayload(), SocketMessageDTO.class); + AskService askService = SpringBeanUtil.getBean(AskService.class); + askService.handlerMessageBySocket(socketMessageDTO); } @Override