package com.supervision.service.impl; import cn.hutool.core.codec.Base64Encoder; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.io.IoUtil; import cn.hutool.core.lang.Assert; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.crypto.digest.MD5; import cn.hutool.json.JSONUtil; import com.supervision.domain.GlobalResult; import com.supervision.exception.BusinessException; import com.supervision.feign.AskQaSimilarityFeignClient; import com.supervision.feign.RasaManageFeignClient; import com.supervision.model.Process; import com.supervision.model.*; import com.supervision.pojo.qaSimilarity.QaSimilarityQuestion; import com.supervision.pojo.qaSimilarity.QaSimilarityQuestionAnswer; import com.supervision.pojo.vo.TalkResultResVO; import com.supervision.pojo.vo.TalkVideoReqVO; import com.supervision.pojo.vo.TalkVideoTtsResultResVO; import com.supervision.service.*; import com.supervision.util.AsrUtil; import com.supervision.util.MinioUtil; import com.supervision.util.TtsUtil; import com.supervision.util.UserUtil; import com.supervision.vo.rasa.RasaTalkVo; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.io.InputStream; import java.util.Comparator; import java.util.List; import java.util.Optional; @Slf4j @Service @RequiredArgsConstructor public class AskServiceImpl implements AskService { private final ProcessService processService; private final AskTemplateQuestionLibraryService askTemplateQuestionLibraryService; private final AskPatientAnswerService askPatientAnswerService; private final ConfigPhysicalToolService configPhysicalToolService; private final ConfigAncillaryItemService configAncillaryItemService; private final FileResourceService fileResourceService; private final RasaManageFeignClient rasaManageFeignClient; private final AskQaSimilarityFeignClient askQaSimilarityFeignClient; private final CommonDicService commonDicService; private final AiService aiService; private final MedicalRecService medicalRecService; private final DiagnosisAiRecordService diagnosisAiRecordService; @Override public String receiveVoiceFile(MultipartFile file) { if (file.getSize() <= 0) { throw new BusinessException("语音内容为空"); } // 获取音频对应的文字 String text = null; try { text = AsrUtil.asrTransformByBytes(file.getBytes()); } catch (Exception e) { throw new BusinessException("获取语音失败"); } if (StrUtil.isEmpty(text)) { throw new BusinessException("语音内容为空"); } return text; } private void saveQaRecord(String processId, String answerType, String answerId, String question, AskTemplateQuestionLibrary library, String resText) { DiagnosisQaRecord record = new DiagnosisQaRecord(); record.setProcessId(processId); record.setAnswerType(answerType); record.setAnswerId(answerId); if (ObjectUtil.isNotEmpty(library)) { record.setQuestionLibraryId(library.getId()); } record.setQuestion(question); record.setAnswer(resText); record.setCreateUserId(UserUtil.getUser().getId()); record.insert(); } private void setActionRelation(TalkResultResVO talkResultResVO) { Integer type = talkResultResVO.getType(); String actionId = talkResultResVO.getActionId(); if (StrUtil.isEmpty(actionId) || null == type) { return; } // 体格检查 if (type.equals(2)) { ConfigPhysicalTool configPhysicalTool = configPhysicalToolService.getById(actionId); if (null != configPhysicalTool) { talkResultResVO.setItemName(configPhysicalTool.getToolName()); talkResultResVO.setItemImage(configPhysicalTool.getIconBase64()); talkResultResVO.setRequireLocation(configPhysicalTool.getRequireLocation()); // 生成树的时候,没有ID字段,所以用名称生成md5作为ID talkResultResVO.setActionType(new MD5().digestHex16(configPhysicalTool.getType())); } } // 辅助检查 if (type.equals(3)) { ConfigAncillaryItem configAncillaryItem = configAncillaryItemService.getById(actionId); if (null != configAncillaryItem) { talkResultResVO.setItemName(configAncillaryItem.getItemName()); // 生成树的时候,没有ID字段,所以用名称生成md5作为ID talkResultResVO.setActionType(new MD5().digestHex16(configAncillaryItem.getType())); } } } @Override public TalkResultResVO talkByVideo(TalkVideoReqVO talkReqVO) { // 根据processId找到对应的病人 Process process = Optional.ofNullable(processService.getById(talkReqVO.getProcessId())).orElseThrow(() -> new BusinessException("未找到诊疗进程")); String talkResult = talkQaSimilarity(talkReqVO.getText(), UserUtil.getUser().getId()); // 调用rasa获取文字内容 //String rasaResult = talkRasa(talkReqVO.getText(), UserUtil.getUser().getId()); log.info("调用对话的回复是:{}", talkResult); TalkResultResVO talkResultResVO = new TalkResultResVO(); // 如果rasa没有识别出来,则返回默认值 if (StrUtil.isBlank(talkResult)) { AskPatientAnswer medicalRecErrorAnswer = getMedicalRecErrorAnswer(process.getMedicalRecId()); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(medicalRecErrorAnswer.getAnswerResourceId())); talkResultResVO.setAnswerMessage(medicalRecErrorAnswer.getAnswer()); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, medicalRecErrorAnswer.getAnswer()); talkResultResVO.setType(1); return talkResultResVO; } // 这里校验,rasa回复的结果是不是action // 这里设置的模板,对于action的动作全部是用ancillary_ | tool_进行标记,详情看生成rasa的yml的代码:RasaServiceImpl.generateDomain // ancillary_ | tool_ if (talkResult.startsWith("ancillary_") || talkResult.startsWith("tool_")) { log.info("呼出语句:{}", talkResult); List actionList = StrUtil.split(talkResult, '_'); if (actionList.size() > 1) { // 在这里设置为动作 talkResultResVO.setActionId(actionList.get(1)); talkResultResVO.setType("ancillary".equals(actionList.get(0)) ? 3 : 2); setActionRelation(talkResultResVO); return talkResultResVO; } } else { AskTemplateQuestionLibrary library = askTemplateQuestionLibraryService.getById(talkResult); if (ObjectUtil.isEmpty(library)) { log.info("{}:未从问题库中找到,回答未识别语句", talkResult); AskPatientAnswer medicalRecErrorAnswer = getMedicalRecErrorAnswer(process.getMedicalRecId()); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(medicalRecErrorAnswer.getAnswerResourceId())); talkResultResVO.setAnswerMessage(medicalRecErrorAnswer.getAnswer()); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, medicalRecErrorAnswer.getAnswer()); } else { AskPatientAnswer askPatientAnswer = askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, process.getMedicalRecId()) .eq(AskPatientAnswer::getLibraryQuestionId, library.getId()).last("limit 1").one(); if (ObjectUtil.isNotEmpty(askPatientAnswer) && StrUtil.isNotEmpty(askPatientAnswer.getAnswerResourceId())) { String resText = askPatientAnswer.getAnswer(); log.info("{}:找到了病历配置的回答语句:{},回答内容:{}", talkResult, askPatientAnswer.getId(), resText); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(askPatientAnswer.getAnswerResourceId())); talkResultResVO.setAnswerMessage(askPatientAnswer.getAnswer()); // 保存记录 saveQaRecord(talkReqVO.getProcessId(), "patient", askPatientAnswer.getId(), talkReqVO.getText(), library, resText); } else { // 临时解决方案,回答没有,在问题库中内置这个回答 log.info("{}:病历配置,从AskPatientAnswer中未找到回答结果,回复 默认回复", talkResult); AskPatientAnswer medicalRecDefaultAnswer = getMedicalRecDefaultAnswer(process.getMedicalRecId()); talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(medicalRecDefaultAnswer.getAnswerResourceId())); talkResultResVO.setAnswerMessage(medicalRecDefaultAnswer.getAnswer()); saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), library, medicalRecDefaultAnswer.getAnswer()); } } } talkResultResVO.setType(1); return talkResultResVO; } /** * rasa接口,不好用,现在已废弃 */ @Deprecated public String talkRasa(String question, String sessionId) { RasaTalkVo rasaTalkVo = new RasaTalkVo(); rasaTalkVo.setQuestion(question); rasaTalkVo.setSessionId(sessionId); // 默认为1 rasaTalkVo.setModelId("1"); try { GlobalResult> talk = rasaManageFeignClient.talk(rasaTalkVo); log.info("调用rasa对话返回结果:{}", talk); if (talk.getCode() != 200 || CollUtil.isEmpty(talk.getData())) { return null; } return CollUtil.getFirst(talk.getData()); } catch (Exception e) { log.error("talkRasa error ", e); } return null; } public String talkQaSimilarity(String question, String sessionId) { log.info("开始调用talkQaSimilarity,问题:{}", question); try { GlobalResult> result = askQaSimilarityFeignClient.askQuestionSimilarityAnswer(new QaSimilarityQuestion(question)); log.info("调用talkQaSimilarity结束,问题:{},返回结果:{}", question, JSONUtil.toJsonStr(result)); return CollUtil.getFirst(result.getData()).getMatchQuestionCode(); } catch (Exception e) { log.error("调用talkQaSimilarity error ", e); return null; } } public QaSimilarityQuestionAnswer talkQaSimilarityWithScore(String question, String sessionId) { log.info("开始调用talkQaSimilarity,问题:{}", question); try { GlobalResult> result = askQaSimilarityFeignClient.askQuestionSimilarityAnswer(new QaSimilarityQuestion(question)); // 排序,降序,取最高的 result.getData().sort(Comparator.comparing(QaSimilarityQuestionAnswer::getMatchScore).reversed()); log.info("调用talkQaSimilarity结束,问题:{},返回结果:{}", question, JSONUtil.toJsonStr(result)); return CollUtil.getFirst(result.getData()); } catch (Exception e) { log.error("调用talkQaSimilarity error ", e); return null; } } private AskPatientAnswer getMedicalRecErrorAnswer(String medicalRecId) { //Optional.ofNullable(medicalRecErrorAnswer).orElseGet(() ->new AskPatientAnswer()).getAnswer() Assert.notEmpty(medicalRecId, "病历id不能为空"); CommonDic commonDic = commonDicService.lambdaQuery().eq(CommonDic::getGroupCode, "AQT").eq(CommonDic::getCode, "system_error").one(); Assert.notNull(commonDic, "字典未配置系统内置异常识别场景"); AskTemplateQuestionLibrary askTemplateQuestionLibrary = askTemplateQuestionLibraryService.lambdaQuery().eq(AskTemplateQuestionLibrary::getDictId, commonDic.getId()).one(); Assert.notNull(askTemplateQuestionLibrary, "知识库未配置系统内置异常识别场景"); return askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, medicalRecId).eq(AskPatientAnswer::getLibraryQuestionId, askTemplateQuestionLibrary.getId()).one(); } private AskPatientAnswer getMedicalRecDefaultAnswer(String medicalRecId) { //Optional.ofNullable(medicalRecErrorAnswer).orElseGet(() ->new AskPatientAnswer()).getAnswer() Assert.notEmpty(medicalRecId, "病历id不能为空"); CommonDic commonDic = commonDicService.lambdaQuery().eq(CommonDic::getGroupCode, "AQT").eq(CommonDic::getCode, "system_default").one(); Assert.notNull(commonDic, "字典未配置默认回答场景"); AskTemplateQuestionLibrary askTemplateQuestionLibrary = askTemplateQuestionLibraryService.lambdaQuery().eq(AskTemplateQuestionLibrary::getDictId, commonDic.getId()).one(); Assert.notNull(askTemplateQuestionLibrary, "知识库未配置系统内置异常识别场景"); return askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, medicalRecId).eq(AskPatientAnswer::getLibraryQuestionId, askTemplateQuestionLibrary.getId()).one(); } /** * 获取应答视频的base64位编码 * * @param fileResourceId 应答id * @return 应答视频base64位编码 */ private String getAnswerVideoBase64OrDefault(String fileResourceId) { FileResource fileResource = fileResourceService.getById(fileResourceId); try (InputStream inputStream = MinioUtil.download(fileResource.getMinioId())) { return Base64Encoder.encode(IoUtil.readBytes(inputStream)); } catch (Exception e) { log.error("获取视频失败", e); throw new BusinessException("未找到回复视频"); } } /** * 使用无声视频+语音转文字的形式来做 * * @param talkReqVO 请求 * @return 返回结果 */ @Override public TalkVideoTtsResultResVO talkByVideoAndTts(TalkVideoReqVO talkReqVO) { // 根据processId找到对应的病人 Process process = Optional.ofNullable(processService.getById(talkReqVO.getProcessId())).orElseThrow(() -> new BusinessException("未找到诊疗进程")); MedicalRec medicalRec = medicalRecService.getById(process.getMedicalRecId()); QaSimilarityQuestionAnswer qaSimilarityQuestionAnswer = talkQaSimilarityWithScore(talkReqVO.getText(), UserUtil.getUser().getId()); TalkVideoTtsResultResVO talkVideoTtsResultResVO = new TalkVideoTtsResultResVO(); // 如果匹配度没有匹配到任何数据,则走大模型 if (ObjectUtil.isEmpty(qaSimilarityQuestionAnswer)) { String talk = aiService.talk(talkReqVO.getText(), medicalRec.getMedicalRecordAi()); talkVideoTtsResultResVO.setAnswerMessage(talk); saveAiRecord(process.getId(), talkReqVO.getText(), talkVideoTtsResultResVO.getAnswerMessage()); } else { // 如果阈值过低,也走大模型 if (qaSimilarityQuestionAnswer.getMatchScore() < 0.5) { log.info("{}:匹配到的结果阈值过低,走大模型回答", qaSimilarityQuestionAnswer); String talk = aiService.talk(talkReqVO.getText(), medicalRec.getMedicalRecordAi()); talkVideoTtsResultResVO.setAnswerMessage(talk); saveAiRecord(process.getId(), talkReqVO.getText(), talkVideoTtsResultResVO.getAnswerMessage()); } else { // 如果查到的问题不在问题库中,走大模型回答 AskTemplateQuestionLibrary library = askTemplateQuestionLibraryService.getById(qaSimilarityQuestionAnswer.getMatchQuestionCode()); if (ObjectUtil.isEmpty(library)) { log.info("{}:未从问题库中找到,走大模型回答", qaSimilarityQuestionAnswer); String talk = aiService.talk(talkReqVO.getText(), medicalRec.getMedicalRecordAi()); talkVideoTtsResultResVO.setAnswerMessage(talk); saveAiRecord(process.getId(), talkReqVO.getText(), talkVideoTtsResultResVO.getAnswerMessage()); } else { // 根据问题找这个病历配置的答案 AskPatientAnswer askPatientAnswer = askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, process.getMedicalRecId()) .eq(AskPatientAnswer::getLibraryQuestionId, library.getId()).last("limit 1").one(); // 如果找到了,就走病历配置的内容回答 if (ObjectUtil.isNotEmpty(askPatientAnswer)) { String resText = askPatientAnswer.getAnswer(); log.info("{}:找到了病历配置的回答语句:{},回答内容:{},走病历回答", qaSimilarityQuestionAnswer.getMatchQuestionCode(), askPatientAnswer.getId(), resText); talkVideoTtsResultResVO.setAnswerMessage(resText); // 保存记录到问答记录表 saveQaRecord(talkReqVO.getProcessId(), "patient", askPatientAnswer.getId(), talkReqVO.getText(), library, resText); } else { // 如果问题的答案没有配置,还是走大模型的回答 log.info("{}:病历配置,从AskPatientAnswer中未找到回答结果,走大模型", qaSimilarityQuestionAnswer.getMatchQuestionCode()); String talk = aiService.talk(talkReqVO.getText(), medicalRec.getMedicalRecordAi()); talkVideoTtsResultResVO.setAnswerMessage(talk); saveAiRecord(process.getId(), talkReqVO.getText(), talkVideoTtsResultResVO.getAnswerMessage()); } } } } talkVideoTtsResultResVO.setVoiceBase64(TtsUtil.ttsTransform(talkVideoTtsResultResVO.getAnswerMessage())); return talkVideoTtsResultResVO; } private void saveAiRecord(String processId, String question, String answer) { DiagnosisAiRecord diagnosisAiRecord = new DiagnosisAiRecord(); diagnosisAiRecord.setProcessId(processId); diagnosisAiRecord.setQuestion(question); diagnosisAiRecord.setAnswer(answer); diagnosisAiRecord.setCreateUserId(UserUtil.getUser().getId()); diagnosisAiRecord.setUpdateUserId(UserUtil.getUser().getId()); diagnosisAiRecordService.save(diagnosisAiRecord); } }