diff --git a/virtual-patient-manage/src/main/java/com/supervision/manage/service/impl/MedicalRecManageServiceImpl.java b/virtual-patient-manage/src/main/java/com/supervision/manage/service/impl/MedicalRecManageServiceImpl.java index 50b32845..ee835dc2 100644 --- a/virtual-patient-manage/src/main/java/com/supervision/manage/service/impl/MedicalRecManageServiceImpl.java +++ b/virtual-patient-manage/src/main/java/com/supervision/manage/service/impl/MedicalRecManageServiceImpl.java @@ -93,7 +93,7 @@ public class MedicalRecManageServiceImpl implements MedicalRecManageService { } AskPatientAnswer askPatientAnswer = new AskPatientAnswer(); askPatientAnswer.setLibraryQuestionId(medicalRecQaVO.getLibraryQuestionId()); - askPatientAnswer.setAnswer(CollUtil.newArrayList(medicalRecQaVO.getMedicalRecAnswer())); + askPatientAnswer.setAnswer(medicalRecQaVO.getMedicalRecAnswer()); askPatientAnswer.setMedicalId(medicalRec.getId()); askPatientAnswer.setPatientId(patient.getId()); askPatientAnswer.insert(); @@ -177,12 +177,12 @@ public class MedicalRecManageServiceImpl implements MedicalRecManageService { // 存在的从Map中移除掉 AskPatientAnswer existAnswer = existQAMap.remove(medicalRecQaVO.getId()); // 然后进行更新 - existAnswer.setAnswer(CollUtil.newArrayList(medicalRecQaVO.getMedicalRecAnswer())); + existAnswer.setAnswer(medicalRecQaVO.getMedicalRecAnswer()); existAnswer.updateById(); } else { AskPatientAnswer askPatientAnswer = new AskPatientAnswer(); askPatientAnswer.setLibraryQuestionId(medicalRecQaVO.getLibraryQuestionId()); - askPatientAnswer.setAnswer(CollUtil.newArrayList(medicalRecQaVO.getMedicalRecAnswer())); + askPatientAnswer.setAnswer(medicalRecQaVO.getMedicalRecAnswer()); askPatientAnswer.setMedicalId(medicalRec.getId()); askPatientAnswer.setPatientId(patient.getId()); askPatientAnswer.insert(); @@ -221,7 +221,7 @@ public class MedicalRecManageServiceImpl implements MedicalRecManageService { medicalRecQaVO.setLibraryQuestionId(e.getLibraryQuestionId()); // 病历的回答 - medicalRecQaVO.setMedicalRecAnswer(CollUtil.getFirst(e.getAnswer())); + medicalRecQaVO.setMedicalRecAnswer(e.getAnswer()); return medicalRecQaVO; }).collect(Collectors.toList()); // 开始构建问题 diff --git a/virtual-patient-model/src/main/java/com/supervision/model/AskPatientAnswer.java b/virtual-patient-model/src/main/java/com/supervision/model/AskPatientAnswer.java index b504d211..bdc2ff02 100644 --- a/virtual-patient-model/src/main/java/com/supervision/model/AskPatientAnswer.java +++ b/virtual-patient-model/src/main/java/com/supervision/model/AskPatientAnswer.java @@ -45,8 +45,10 @@ public class AskPatientAnswer extends Model implements Seriali * 回答 */ @ApiModelProperty(" 回答") - @TableField(typeHandler = StringListTypeHandler.class) - private List answer; + private String answer; + + @ApiModelProperty("回答资源id") + private String answerResourceId; /** * 创建人ID diff --git a/virtual-patient-model/src/main/resources/mapper/AskPatientAnswerMapper.xml b/virtual-patient-model/src/main/resources/mapper/AskPatientAnswerMapper.xml index 01481ef2..e79f4bbc 100644 --- a/virtual-patient-model/src/main/resources/mapper/AskPatientAnswerMapper.xml +++ b/virtual-patient-model/src/main/resources/mapper/AskPatientAnswerMapper.xml @@ -9,8 +9,8 @@ - + + @@ -19,7 +19,7 @@ id,patient_id,medical_id,library_question_id, - answer,create_user_id, + answer,answer_resources_id,create_user_id, create_time,update_user_id,update_time diff --git a/virtual-patient-web/src/main/java/com/supervision/controller/AskController.java b/virtual-patient-web/src/main/java/com/supervision/controller/AskController.java index ed996a53..c7a226ea 100644 --- a/virtual-patient-web/src/main/java/com/supervision/controller/AskController.java +++ b/virtual-patient-web/src/main/java/com/supervision/controller/AskController.java @@ -10,7 +10,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.io.OutputStream; @Api(tags = "问诊") @RestController @@ -32,6 +34,12 @@ public class AskController { return askService.talk(talkReqVO); } + @ApiOperation("使用本地视频的形式来做") + @PostMapping("/talkByVideo") + public TalkResultResVO talkByVideo(@RequestBody TalkReqVO talkReqVO, HttpServletResponse response) throws IOException { + return askService.talkByVideo(talkReqVO,response); + } + } diff --git a/virtual-patient-web/src/main/java/com/supervision/service/AskService.java b/virtual-patient-web/src/main/java/com/supervision/service/AskService.java index cfafcad8..290e5585 100644 --- a/virtual-patient-web/src/main/java/com/supervision/service/AskService.java +++ b/virtual-patient-web/src/main/java/com/supervision/service/AskService.java @@ -2,10 +2,13 @@ package com.supervision.service; import com.supervision.pojo.vo.TalkReqVO; import com.supervision.pojo.vo.TalkResultResVO; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.multipart.MultipartFile; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.io.OutputStream; public interface AskService { @@ -13,7 +16,7 @@ public interface AskService { TalkResultResVO talk(TalkReqVO talkReqVO) throws IOException; - String replyVoice(); + TalkResultResVO talkByVideo(TalkReqVO talkReqVO, HttpServletResponse response) throws IOException; } diff --git a/virtual-patient-web/src/main/java/com/supervision/service/impl/AskServiceImpl.java b/virtual-patient-web/src/main/java/com/supervision/service/impl/AskServiceImpl.java index 876691a2..8f4f17f6 100644 --- a/virtual-patient-web/src/main/java/com/supervision/service/impl/AskServiceImpl.java +++ b/virtual-patient-web/src/main/java/com/supervision/service/impl/AskServiceImpl.java @@ -1,6 +1,7 @@ package com.supervision.service.impl; import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.io.IoUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.RandomUtil; import cn.hutool.core.util.StrUtil; @@ -18,6 +19,9 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.OutputStream; import java.util.List; import java.util.Optional; @@ -109,7 +113,7 @@ public class AskServiceImpl implements AskService { saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), library, "您好,我没有听懂您说什么"); } } else { - if (CollUtil.isEmpty(askPatientAnswer.getAnswer())) { + if (StrUtil.isEmpty(askPatientAnswer.getAnswer())) { log.info("{}:病历配置的回答:{}:为空不为空,但在获取的时候,答案为空,尝试回复默认语句", rasaResult, askPatientAnswer.getId()); if (StrUtil.isNotEmpty(library.getDefaultAnswer())) { String resText = library.getDefaultAnswer(); @@ -123,7 +127,7 @@ public class AskServiceImpl implements AskService { } } else { - String resText = askPatientAnswer.getAnswer().get(RandomUtil.randomInt(0, askPatientAnswer.getAnswer().size())); + String resText = askPatientAnswer.getAnswer(); log.info("{}:找到了病历配置的回答语句:{},回答内容:{}", rasaResult, askPatientAnswer.getId(), resText); HumanUtil.textDriven(resText, roomId); // 保存记录 @@ -152,13 +156,6 @@ public class AskServiceImpl implements AskService { } - @Override - public String replyVoice() { - String text = "测试:这是文字转语音的测试,测试是否OK"; - return TtsUtil.ttsTransform(text); - } - - private void setActionRelation(TalkResultResVO talkResultResVO) { Integer type = talkResultResVO.getType(); String actionId = talkResultResVO.getActionId(); @@ -190,4 +187,95 @@ public class AskServiceImpl implements AskService { } + @Override + public TalkResultResVO talkByVideo(TalkReqVO talkReqVO, HttpServletResponse response) throws IOException { + // 根据processId找到对应的病人 + Process process = Optional.ofNullable(processService.getById(talkReqVO.getProcessId())).orElseThrow(() -> new BusinessException("未找到诊疗进程")); + // 调用rasa获取文字内容 + String rasaResult = RasaUtil.talkRasa(talkReqVO.getText(), UserUtil.getUser().getId(), process.getPatientId()); + // 如果rasa没有识别出来,则返回默认值 + if (StrUtil.isBlank(rasaResult)) { + // TODO 这里应该返回视频流 + String roomId = HumanUtil.queryRoomId(talkReqVO.getRoomKey(), talkReqVO.getRoomToken()); + HumanUtil.textDriven("您好,我没有听懂您说什么", roomId); + + saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, "您好,我没有听懂您说什么"); + } + TalkResultResVO talkResultResVO = new TalkResultResVO(); + // 这里校验,rasa回复的结果是不是action + // 这里设置的模板,对于action的动作全部是用ancillary_ | tool_进行标记,详情看生成rasa的yml的代码:RasaServiceImpl.generateDomain + // ancillary_ | tool_ + if (rasaResult.startsWith("ancillary_") || rasaResult.startsWith("tool_")) { + log.info("呼出语句:{}", rasaResult); + List actionList = StrUtil.split(rasaResult, '_'); + if (actionList.size() > 1) { + // 在这里给socket回复,设置为动作 + talkResultResVO.setActionId(actionList.get(1)); + talkResultResVO.setType("ancillary".equals(actionList.get(0)) ? 3 : 2); + setActionRelation(talkResultResVO); + return talkResultResVO; + } + } else { + String roomId = HumanUtil.queryRoomId(talkReqVO.getRoomKey(), talkReqVO.getRoomToken()); + AskTemplateQuestionLibrary library = askTemplateQuestionLibraryService.getById(rasaResult); + if (ObjectUtil.isEmpty(library)) { + log.info("{}:未从问题库中找到,回答未识别语句", rasaResult); + // TODO 这里应该返回视频 + HumanUtil.textDriven("您好,我没有听懂您说什么", roomId); + + saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, "您好,我没有听懂您说什么"); + } else { + AskPatientAnswer askPatientAnswer = askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getPatientId, process.getPatientId()) + .eq(AskPatientAnswer::getLibraryQuestionId, library.getId()).last("limit 1").one(); + // 如果没有找到回答,去默认回答里面看看有没有 + if (ObjectUtil.isEmpty(askPatientAnswer)) { + log.info("{}:病历配置,未从AskPatientAnswer中找到回答结果,尝试回答默认答案", rasaResult); + // 首先看看default里面是不是存在,如果存在,就从default里面去找 + if (StrUtil.isNotEmpty(library.getDefaultAnswer())) { + String resText = library.getDefaultAnswer(); + // TODO 这里应该返回视频 + + HumanUtil.textDriven(resText, roomId); + // 保存记录 + saveQaRecord(talkReqVO.getProcessId(), "default", library.getId(), talkReqVO.getText(), library, resText); + log.info("{}:找到了默认答案:{}", rasaResult, talkReqVO.getText()); + } else { + log.info("{}:没有从默认答案中找到找到默认内容,回复未识别语句", rasaResult); + HumanUtil.textDriven("您好,我没有听懂您说什么", roomId); + saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), library, "您好,我没有听懂您说什么"); + } + } else { + if (StrUtil.isEmpty(askPatientAnswer.getAnswer())) { + log.info("{}:病历配置的回答:{}:为空不为空,但在获取的时候,答案为空,尝试回复默认语句", rasaResult, askPatientAnswer.getId()); + if (StrUtil.isNotEmpty(library.getDefaultAnswer())) { + String resText = library.getDefaultAnswer(); + log.info("{}:病历配置的回答:{}:为空不为空不为空,但在获取的时候,答案为空,开始回复默认语句,默认语句内容:{}", rasaResult, askPatientAnswer.getId(), resText); + // TODO 这里应该返回视频 + HumanUtil.textDriven(resText, roomId); + // 保存记录 + saveQaRecord(talkReqVO.getProcessId(), "default", library.getId(), talkReqVO.getText(), library, resText); + } else { + log.info("{}:病历配置的回答:{}:为空不为空,但在获取的时候,答案为空,但是获取默认语句也为空,那么回复未识别语句", rasaResult, askPatientAnswer.getId()); + // TODO 这里应该返回视频 + HumanUtil.textDriven("您好,我没有听懂您说什么", roomId); + } + + } else { + String resText = askPatientAnswer.getAnswer(); + log.info("{}:找到了病历配置的回答语句:{},回答内容:{}", rasaResult, askPatientAnswer.getId(), resText); + // TODO 这里应该返回视频 + HumanUtil.textDriven(resText, roomId); + // 保存记录 + saveQaRecord(talkReqVO.getProcessId(), "patient", askPatientAnswer.getId(), talkReqVO.getText(), library, resText); + } + + } + } + } + talkResultResVO.setType(1); + return talkResultResVO; + +// IoUtil.copy(inputStream, response.getOutputStream()); +// return null; + } }