Yaxin
/
virtual-patient
31
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

问诊socket流程提交

Browse Source
dev_v1.0.1
liu 2 years ago
parent acf4256904
commit b2848b8ccd
8 changed files with 67 additions and 43 deletions
Show Stats Download Patch File Download Diff File

4
virtual-patient-web/src/main/java/com/supervision/controller/AskController.java
Unescape Escape View File

@ -1,11 +1,9 @@
package com.supervision.controller;
import com.supervision.pojo.vo.ReplyVoiceResVO;
import com.supervision.service.AskService;
import com.supervision.websocket.cache.WebSocketUserCache;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
@ -39,7 +37,7 @@ public class AskController {
@ApiOperation("回复语音及文字消息")
@GetMapping("replyVoice")
public ReplyVoiceResVO replyVoice() {
public String replyVoice() {
return askService.replyVoice();
}

8
virtual-patient-web/src/main/java/com/supervision/paddlespeech/TtsUtil.java
Unescape Escape View File

@ -9,7 +9,6 @@ import com.supervision.exception.BusinessException;
import com.supervision.paddlespeech.dto.req.TtsReqDTO;
import com.supervision.paddlespeech.dto.res.PaddleSpeechResDTO;
import com.supervision.paddlespeech.dto.res.TtsResultDTO;
import com.supervision.pojo.vo.ReplyVoiceResVO;
import com.supervision.util.SpringBeanUtil;
import org.springframework.core.env.Environment;
@ -19,7 +18,7 @@ public class TtsUtil {
private static final ObjectMapper objectMapper = new ObjectMapper();
public static ReplyVoiceResVO ttsTransform(String str) {
public static String ttsTransform(String str) {
// 构建
String post = HttpUtil.post(TTS_URL, JSONUtil.toJsonStr(new TtsReqDTO(str)));
try {
@ -28,10 +27,7 @@ public class TtsUtil {
if (!response.getSuccess() || ObjectUtil.isEmpty(response.getResult())) {
throw new BusinessException("文字转换语音失败");
}
ReplyVoiceResVO resVO = new ReplyVoiceResVO();
resVO.setVoice(response.getResult().getAudio());
resVO.setText(str);
return resVO;
return response.getResult().getAudio();
} catch (Exception e) {
throw new BusinessException("语音转换文字失败");
}

11
virtual-patient-web/src/main/java/com/supervision/pojo/vo/ReplyVoiceResVO.java
Unescape Escape View File

@ -1,11 +0,0 @@
package com.supervision.pojo.vo;
import lombok.Data;
@Data
public class ReplyVoiceResVO {
private String voice;
private String text;
}

7
virtual-patient-web/src/main/java/com/supervision/service/AskService.java
Unescape Escape View File

@ -1,9 +1,6 @@
package com.supervision.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.supervision.pojo.vo.ReplyVoiceResVO;
import com.supervision.websocket.dto.SocketMessageDTO;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
@ -11,11 +8,11 @@ import java.util.List;
public interface AskService {
void handlerMessageBySocket(SocketMessageDTO socketMessageDTO);
void handlerMessageBySocket(SocketMessageDTO socketMessageDTO) throws IOException;
String receiveVoiceFile(MultipartFile file) throws IOException;
ReplyVoiceResVO replyVoice();
String replyVoice();
List<String> conversation(String question, String sessionId);

66
virtual-patient-web/src/main/java/com/supervision/service/impl/AskServiceImpl.java
Unescape Escape View File

@ -1,25 +1,25 @@
package com.supervision.service.impl;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONUtil;
import com.supervision.exception.BusinessException;
import com.supervision.model.User;
import com.supervision.paddlespeech.AsrUtil;
import com.supervision.paddlespeech.TtsUtil;
import com.supervision.rasa.RasaUtil;
import com.supervision.rasa.dto.RasaReqDTO;
import com.supervision.rasa.dto.RasaResDTO;
import com.supervision.pojo.vo.ReplyVoiceResVO;
import com.supervision.service.AskService;
import com.supervision.util.UserUtil;
import com.supervision.websocket.cache.WebSocketUserCache;
import com.supervision.websocket.dto.ActionDTO;
import com.supervision.websocket.dto.SocketMessageDTO;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.socket.TextMessage;
import org.springframework.web.socket.WebSocketSession;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
@ -27,7 +27,7 @@ public class AskServiceImpl implements AskService {
@Override
public void handlerMessageBySocket(SocketMessageDTO socketMessageDTO) {
public void handlerMessageBySocket(SocketMessageDTO socketMessageDTO) throws IOException {
// 首先获取消息的类型
String text;
if (0 == socketMessageDTO.getMessageType()) {
@ -41,8 +41,52 @@ public class AskServiceImpl implements AskService {
throw new BusinessException("语音消息不能为空");
}
// 进行rasa对话
List<String> rasaResultList = RasaUtil.talkRasa(text, socketMessageDTO.getSocketId());
WebSocketSession session = WebSocketUserCache.getSession(socketMessageDTO.getSocketId());
for (String rasaResult : rasaResultList) {
if (StrUtil.isNotBlank(rasaResult)){
// 这里校验,rasa回复的结果是不是action
// 这里设置的模板,对于action的动作全部是用---进行标记,详情看生成rasa的yml的代码:RasaServiceImpl.generateDomain
// ---ancillary---xxx
// ---tool---xxx
if (rasaResult.startsWith("---")){
// ["","ancillary","xxx"]
List<String> actionList = StrUtil.split(rasaResult, "---");
if (actionList.size() > 2){
ActionDTO actionDTO = new ActionDTO();
actionDTO.setActionType(actionList.get(1));
actionDTO.setActionId(actionList.get(2));
// 在这里给socket回复,设置为动作
SocketMessageDTO res = new SocketMessageDTO();
res.setSocketId(socketMessageDTO.getSocketId());
res.setUserId(UserUtil.getUser().getId());
res.setAction(actionDTO);
res.setType(2);
session.sendMessage(new TextMessage(JSONUtil.toJsonStr(res)));
return;
}
}else {
// 走到这里,说明是文字,这个时候文字转语音
String replyVoiceResVO = TtsUtil.ttsTransform(rasaResult);
// 在这里给socket回复
SocketMessageDTO res = new SocketMessageDTO();
res.setSocketId(socketMessageDTO.getSocketId());
res.setUserId(UserUtil.getUser().getId());
res.setTextMessage(rasaResult);
res.setVoiceMessage(replyVoiceResVO);
res.setType(1);
session.sendMessage(new TextMessage(JSONUtil.toJsonStr(res)));
return;
}
}
}
// 兜底,如果走到了这里,就直接返回未识别
SocketMessageDTO res = new SocketMessageDTO();
res.setSocketId(socketMessageDTO.getSocketId());
res.setUserId(UserUtil.getUser().getId());
res.setTextMessage("医生,我们有听懂您说的什么");
res.setType(1);
session.sendMessage(new TextMessage(JSONUtil.toJsonStr(res)));
}
@Override
@ -55,7 +99,7 @@ public class AskServiceImpl implements AskService {
@Override
public ReplyVoiceResVO replyVoice() {
public String replyVoice() {
String text = "测试:这是文字转语音的测试,测试是否OK";
return TtsUtil.ttsTransform(text);
}
@ -63,6 +107,6 @@ public class AskServiceImpl implements AskService {
@Override
public List<String> conversation(String question, String sessionId) {
return RasaUtil.talkRasa(question,sessionId);
return RasaUtil.talkRasa(question, sessionId);
}
}

4
virtual-patient-web/src/main/java/com/supervision/service/impl/RasaServiceImpl.java
Unescape Escape View File

@ -168,12 +168,10 @@ public class RasaServiceImpl implements RasaService {
itemCodeIdMap.put(itemIntent, ancillary);
}
// 生成后生成yml对象
// createYmlFile(nluYml, "nlu.yml");
// 加载模板配置
NluYmlTemplate nluYmlTemplate = new NluYmlTemplate();
nluYmlTemplate.setNlu(nluList);
// 生成后生成yml文件
createYmlFile(NluYmlTemplate.class, "nlu.ftl", nluYmlTemplate, "nlu.yml", ymalFileMap);
}

6
virtual-patient-web/src/main/java/com/supervision/websocket/dto/SocketMessageDTO.java
Unescape Escape View File

@ -24,14 +24,14 @@ public class SocketMessageDTO {
private ActionDTO action;
/**
* 后端返回给前端时使用,表示该是消息还是action动作,0消息,1动作
* 后端返回给前端时使用,表示该是消息还是action动作,1消息,2动作
*/
@ApiModelProperty("后端返回给前端时使用,表示该是消息还是action动作,0消息,1动作")
@ApiModelProperty("后端返回给前端时使用,表示该是消息还是action动作,1消息,2动作")
private Integer type;
/**
* 表示是消息还是action动作,0语音,1文字
*/
@ApiModelProperty("前端到后端使用,表示是语音还是文字,0语音,1文字")
@ApiModelProperty("前端到后端使用,表示是语音还是文字,1语音,2文字")
private Integer messageType;
}

4
virtual-patient-web/src/main/java/com/supervision/websocket/handler/AskWebSocketHandler.java
Unescape Escape View File

@ -12,6 +12,8 @@ import org.springframework.web.socket.TextMessage;
import org.springframework.web.socket.WebSocketSession;
import org.springframework.web.socket.handler.TextWebSocketHandler;
import java.io.IOException;
@Slf4j
public class AskWebSocketHandler extends TextWebSocketHandler {
@ -28,7 +30,7 @@ public class AskWebSocketHandler extends TextWebSocketHandler {
}
@Override
protected void handleTextMessage(WebSocketSession session, TextMessage message) {
protected void handleTextMessage(WebSocketSession session, TextMessage message) throws IOException {
// 处理接收到的消息
log.info("收到消息:{}", message.toString());
// 这里反序列化消息,将消息形成固定的格式

Write Preview
Loading…
Cancel
Save