package com.supervision.service.impl; import cn.hutool.core.codec.Base64Encoder; import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.crypto.digest.MD5; import cn.hutool.setting.dialect.Props; import com.supervision.exception.BusinessException; import com.supervision.model.Process; import com.supervision.model.*; import com.supervision.pojo.vo.TalkReqVO; import com.supervision.pojo.vo.TalkResultResVO; import com.supervision.pojo.vo.TalkVideoReqVO; import com.supervision.service.*; import com.supervision.util.*; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Optional; @Slf4j @Service @RequiredArgsConstructor public class AskServiceImpl implements AskService { private final ProcessService processService; private final AskTemplateQuestionLibraryService askTemplateQuestionLibraryService; private final AskPatientAnswerService askPatientAnswerService; private final ConfigPhysicalToolService configPhysicalToolService; private final ConfigAncillaryItemService configAncillaryItemService; private final FileResourceService fileResourceService; @Value("${answer.defaultNoMatchId}") private String defaultNoMatchId; @Value("${answer.video.basePath}") private String videoPath; @Value("${video-url}") private String videoUrl; @Override public String receiveVoiceFile(MultipartFile file) { if (file.getSize() <= 0) { throw new BusinessException("语音内容为空"); } // 获取音频对应的文字 String text = null; try { text = AsrUtil.asrTransformByBytes(file.getBytes()); } catch (Exception e) { throw new BusinessException("获取语音失败"); } if (StrUtil.isEmpty(text)) { throw new BusinessException("语音内容为空"); } return text; } @Override @Transactional(rollbackFor = Exception.class) public TalkResultResVO talk(TalkReqVO talkReqVO) { // 根据processId找到对应的病人 Process process = Optional.ofNullable(processService.getById(talkReqVO.getProcessId())).orElseThrow(() -> new BusinessException("未找到诊疗进程")); // 调用rasa获取文字内容 String rasaResult = RasaUtil.talkRasa(talkReqVO.getText(), UserUtil.getUser().getId()); // 如果rasa没有识别出来,则返回默认值 if (StrUtil.isBlank(rasaResult)) { // 这里调用京东数字人接口首先根据token获取房间号 String roomId = HumanUtil.queryRoomId(talkReqVO.getRoomKey(), talkReqVO.getRoomToken()); HumanUtil.textDriven("您好,我没有听懂您说什么", roomId); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, "您好,我没有听懂您说什么"); } TalkResultResVO talkResultResVO = new TalkResultResVO(); // 这里校验,rasa回复的结果是不是action // 这里设置的模板,对于action的动作全部是用ancillary_ | tool_进行标记,详情看生成rasa的yml的代码:RasaServiceImpl.generateDomain // ancillary_ | tool_ if (rasaResult.startsWith("ancillary_") || rasaResult.startsWith("tool_")) { log.info("呼出语句:{}", rasaResult); List actionList = StrUtil.split(rasaResult, '_'); if (actionList.size() > 1) { // 在这里给socket回复,设置为动作 talkResultResVO.setActionId(actionList.get(1)); talkResultResVO.setType("ancillary".equals(actionList.get(0)) ? 3 : 2); setActionRelation(talkResultResVO); return talkResultResVO; } } else { String roomId = HumanUtil.queryRoomId(talkReqVO.getRoomKey(), talkReqVO.getRoomToken()); AskTemplateQuestionLibrary library = askTemplateQuestionLibraryService.getById(rasaResult); if (ObjectUtil.isEmpty(library)) { log.info("{}:未从问题库中找到,回答未识别语句", rasaResult); HumanUtil.textDriven("您好,我没有听懂您说什么", roomId); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, "您好,我没有听懂您说什么"); } else { AskPatientAnswer askPatientAnswer = askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, process.getMedicalRecId()) .eq(AskPatientAnswer::getLibraryQuestionId, library.getId()).last("limit 1").one(); // 如果没有找到回答,去默认回答里面看看有没有 if (ObjectUtil.isEmpty(askPatientAnswer)) { log.info("{}:病历配置,未从AskPatientAnswer中找到回答结果,尝试回答默认答案", rasaResult); // 首先看看default里面是不是存在,如果存在,就从default里面去找 if (StrUtil.isNotEmpty(library.getDefaultAnswer())) { String resText = library.getDefaultAnswer(); HumanUtil.textDriven(resText, roomId); // 保存记录 saveQaRecord(talkReqVO.getProcessId(), "default", library.getId(), talkReqVO.getText(), library, resText); log.info("{}:找到了默认答案:{}", rasaResult, talkReqVO.getText()); } else { log.info("{}:没有从默认答案中找到找到默认内容,回复未识别语句", rasaResult); HumanUtil.textDriven("您好,我没有听懂您说什么", roomId); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), library, "您好,我没有听懂您说什么"); } } else { if (StrUtil.isEmpty(askPatientAnswer.getAnswer())) { log.info("{}:病历配置的回答:{}:为空不为空,但在获取的时候,答案为空,尝试回复默认语句", rasaResult, askPatientAnswer.getId()); if (StrUtil.isNotEmpty(library.getDefaultAnswer())) { String resText = library.getDefaultAnswer(); log.info("{}:病历配置的回答:{}:为空不为空不为空,但在获取的时候,答案为空,开始回复默认语句,默认语句内容:{}", rasaResult, askPatientAnswer.getId(), resText); HumanUtil.textDriven(resText, roomId); // 保存记录 saveQaRecord(talkReqVO.getProcessId(), "default", library.getId(), talkReqVO.getText(), library, resText); } else { log.info("{}:病历配置的回答:{}:为空不为空,但在获取的时候,答案为空,但是获取默认语句也为空,那么回复未识别语句", rasaResult, askPatientAnswer.getId()); HumanUtil.textDriven("您好,我没有听懂您说什么", roomId); } } else { String resText = askPatientAnswer.getAnswer(); log.info("{}:找到了病历配置的回答语句:{},回答内容:{}", rasaResult, askPatientAnswer.getId(), resText); HumanUtil.textDriven(resText, roomId); // 保存记录 saveQaRecord(talkReqVO.getProcessId(), "patient", askPatientAnswer.getId(), talkReqVO.getText(), library, resText); } } } } talkResultResVO.setType(1); return talkResultResVO; } private void saveQaRecord(String processId, String answerType, String answerId, String question, AskTemplateQuestionLibrary library, String resText) { DiagnosisQaRecord record = new DiagnosisQaRecord(); record.setProcessId(processId); record.setAnswerType(answerType); record.setAnswerId(answerId); if (ObjectUtil.isNotEmpty(library)) { record.setQuestionLibraryId(library.getId()); } record.setQuestion(question); record.setAnswer(resText); record.setCreateUserId(UserUtil.getUser().getId()); record.insert(); } private void setActionRelation(TalkResultResVO talkResultResVO) { Integer type = talkResultResVO.getType(); String actionId = talkResultResVO.getActionId(); if (StrUtil.isEmpty(actionId) || null == type) { return; } // 体格检查 if (type.equals(2)) { ConfigPhysicalTool configPhysicalTool = configPhysicalToolService.getById(actionId); if (null != configPhysicalTool) { talkResultResVO.setItemName(configPhysicalTool.getToolName()); talkResultResVO.setItemImage(configPhysicalTool.getIconBase64()); talkResultResVO.setRequireLocation(configPhysicalTool.getRequireLocation()); // 生成树的时候,没有ID字段,所以用名称生成md5作为ID talkResultResVO.setActionType(new MD5().digestHex16(configPhysicalTool.getType())); } } // 辅助检查 if (type.equals(3)) { ConfigAncillaryItem configAncillaryItem = configAncillaryItemService.getById(actionId); if (null != configAncillaryItem) { talkResultResVO.setItemName(configAncillaryItem.getItemName()); // 生成树的时候,没有ID字段,所以用名称生成md5作为ID talkResultResVO.setActionType(new MD5().digestHex16(configAncillaryItem.getType())); } } } @Override public TalkResultResVO talkByVideo(TalkVideoReqVO talkReqVO) throws IOException { // 根据processId找到对应的病人 Process process = Optional.ofNullable(processService.getById(talkReqVO.getProcessId())).orElseThrow(() -> new BusinessException("未找到诊疗进程")); // 调用rasa获取文字内容 String rasaResult = RasaUtil.talkRasa(talkReqVO.getText(), UserUtil.getUser().getId()); log.info("rasa的回复是:{}", rasaResult); TalkResultResVO talkResultResVO = new TalkResultResVO(); // 如果rasa没有识别出来,则返回默认值 if (StrUtil.isBlank(rasaResult)) { talkResultResVO.setVideoUrl(videoUrl + defaultNoMatchId); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(defaultNoMatchId,null)); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, "您好,我没有听懂您说什么"); } // 这里校验,rasa回复的结果是不是action // 这里设置的模板,对于action的动作全部是用ancillary_ | tool_进行标记,详情看生成rasa的yml的代码:RasaServiceImpl.generateDomain // ancillary_ | tool_ if (rasaResult.startsWith("ancillary_") || rasaResult.startsWith("tool_")) { log.info("呼出语句:{}", rasaResult); List actionList = StrUtil.split(rasaResult, '_'); if (actionList.size() > 1) { // 在这里给socket回复,设置为动作 talkResultResVO.setActionId(actionList.get(1)); talkResultResVO.setType("ancillary".equals(actionList.get(0)) ? 3 : 2); setActionRelation(talkResultResVO); return talkResultResVO; } } else { AskTemplateQuestionLibrary library = askTemplateQuestionLibraryService.getById(rasaResult); if (ObjectUtil.isEmpty(library)) { log.info("{}:未从问题库中找到,回答未识别语句", rasaResult); talkResultResVO.setVideoUrl(videoUrl + defaultNoMatchId); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(defaultNoMatchId,null)); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, "您好,我没有听懂您说什么"); } else { AskPatientAnswer askPatientAnswer = askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, process.getMedicalRecId()) .eq(AskPatientAnswer::getLibraryQuestionId, library.getId()).last("limit 1").one(); // 如果没有找到回答,去默认回答里面看看有没有 if (ObjectUtil.isEmpty(askPatientAnswer)) { log.info("{}:病历配置,未从AskPatientAnswer中找到回答结果,尝试回答默认答案", rasaResult); // 首先看看default里面是不是存在,如果存在,就从default里面去找 if (StrUtil.isNotEmpty(library.getDefaultAnswer()) && StrUtil.isNotBlank(library.getDefaultAnswerResourceId())) { String resText = library.getDefaultAnswer(); talkResultResVO.setVideoUrl(videoUrl + library.getDefaultAnswerResourceId()); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(library.getDefaultAnswerResourceId(),null)); // 保存记录 saveQaRecord(talkReqVO.getProcessId(), "default", library.getId(), talkReqVO.getText(), library, resText); log.info("{}:找到了默认答案:{}", rasaResult, talkReqVO.getText()); } else { log.info("{}:没有从默认答案中找到找到默认内容,回复未识别语句", rasaResult); talkResultResVO.setVideoUrl(videoUrl + (StrUtil.isEmpty(library.getDefaultAnswerResourceId()) ? defaultNoMatchId :library.getDefaultAnswerResourceId())); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(library.getDefaultAnswerResourceId(),defaultNoMatchId)); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), library, "您好,我没有听懂您说什么"); } } else { if (StrUtil.isEmpty(askPatientAnswer.getAnswer())) { log.info("{}:病历配置的回答:{}:为空不为空,但在获取的时候,答案为空,尝试回复默认语句", rasaResult, askPatientAnswer.getId()); if (StrUtil.isNotEmpty(library.getDefaultAnswer())) { String resText = library.getDefaultAnswer(); log.info("{}:病历配置的回答:{}:为空不为空不为空,但在获取的时候,答案为空,开始回复默认语句,默认语句内容:{}", rasaResult, askPatientAnswer.getId(), resText); // 这里返回视频 talkResultResVO.setVideoUrl(videoUrl + library.getDefaultAnswerResourceId()); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(library.getDefaultAnswerResourceId(),defaultNoMatchId)); // 保存记录 saveQaRecord(talkReqVO.getProcessId(), "default", library.getId(), talkReqVO.getText(), library, resText); } else { log.info("{}:病历配置的回答:{}:为空不为空,但在获取的时候,答案为空,但是获取默认语句也为空,那么回复未识别语句", rasaResult, askPatientAnswer.getId()); talkResultResVO.setVideoUrl(videoUrl + defaultNoMatchId); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(defaultNoMatchId,null)); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), library, "您好,我没有听懂您说什么"); } } else { String resText = askPatientAnswer.getAnswer(); log.info("{}:找到了病历配置的回答语句:{},回答内容:{}", rasaResult, askPatientAnswer.getId(), resText); talkResultResVO.setVideoUrl(videoUrl + askPatientAnswer.getAnswerResourceId()); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(askPatientAnswer.getAnswerResourceId(),defaultNoMatchId)); // 保存记录 saveQaRecord(talkReqVO.getProcessId(), "patient", askPatientAnswer.getId(), talkReqVO.getText(), library, resText); } } } } talkResultResVO.setType(1); return talkResultResVO; } @Override public void downloadTalkVideo(String fileResourceId, HttpServletResponse response) { FileResource fileResource = fileResourceService.getById(fileResourceId); if (ObjectUtil.isEmpty(fileResource)) { throw new BusinessException("未找到回复视频"); } try (InputStream inputStream = MinioUtil.download(fileResource.getMinioId())) { IoUtil.copy(inputStream, response.getOutputStream()); } catch (Exception e) { log.error("获取视频失败", e); throw new BusinessException("未找到回复视频"); } } /** * 获取应答视频的base64位编码 * @param answerId 应答id * @param defaultValue 默认应答id * @return 应答视频base64位编码 */ private String getAnswerVideoBase64OrDefault(String answerId, String defaultValue) { Props props = new Props(videoPath + File.separator + "index"); String fileName = ""; if (StrUtil.isNotEmpty(answerId) && StrUtil.isNotEmpty(props.getStr(answerId))){ fileName = props.getStr(answerId); }else if (StrUtil.isNotEmpty(defaultValue)){ fileName = props.getStr(defaultValue); } if (StrUtil.isEmpty(fileName)){ return null; } return Base64Encoder.encode(FileUtil.readBytes(videoPath + File.separator + fileName)); } }