package com.supervision.service.impl; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import com.supervision.domain.QaSimilarityQuestionAnswer; import com.supervision.exception.BusinessException; import com.supervision.model.Process; import com.supervision.model.*; import com.supervision.pojo.vo.TalkVideoReqVO; import com.supervision.pojo.vo.TalkVideoTtsResultResVO; import com.supervision.service.*; import com.supervision.util.AsrUtil; import com.supervision.util.SimilarityUtil; import com.supervision.util.TtsUtil; import com.supervision.util.UserUtil; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; @Slf4j @Service @RequiredArgsConstructor public class AskServiceImpl implements AskService { private final ProcessService processService; private final AskTemplateQuestionLibraryService askTemplateQuestionLibraryService; private final AskPatientAnswerService askPatientAnswerService; private final AiService aiService; private final MedicalRecService medicalRecService; private final AskCirculationDetailService askCirculationDetailService; private final CommonDicService commonDicService; @Value("${threshold:0.7}") private String threshold; @Override public String receiveVoiceFile(MultipartFile file) { if (file.getSize() <= 0) { throw new BusinessException("语音内容为空"); } // 获取音频对应的文字 String text = null; try { text = AsrUtil.asrTransformByBytes(file.getBytes()); } catch (Exception e) { throw new BusinessException("获取语音失败"); } if (StrUtil.isEmpty(text)) { throw new BusinessException("语音内容为空"); } return text; } /** * 使用无声视频+语音转文字的形式来做 * * @param talkReqVO 请求 * @return 返回结果 */ @Override public TalkVideoTtsResultResVO talkByVideoAndTts(TalkVideoReqVO talkReqVO) { String answer = talkByVideoAndTts(talkReqVO.getProcessId(), talkReqVO.getText()); TalkVideoTtsResultResVO talkVideoTtsResultResVO = new TalkVideoTtsResultResVO(); talkVideoTtsResultResVO.setVoiceBase64(TtsUtil.ttsTransform(answer)); talkVideoTtsResultResVO.setAnswerMessage(answer); return talkVideoTtsResultResVO; } private String talkByVideoAndTts(String processId, String question) { // 流转记录表 List circulationList = new ArrayList<>(); // 根据processId找到对应的病人 Process process = Optional.ofNullable(processService.getById(processId)).orElseThrow(() -> new BusinessException("未找到诊疗进程")); MedicalRec medicalRec = medicalRecService.getById(process.getMedicalRecId()); // 进行相似度匹配 List similarityAnswerList = SimilarityUtil.talkRedisVectorWithScore(question); Optional first = similarityAnswerList.stream().findFirst(); // 如果匹配度没有匹配到任何数据 if (first.isEmpty()) { // 记录流转信息 circulationList.add(AskCirculationDetail.builder().failInfo("相似度返回内容为空,走大模型").build()); // 如果没有匹配到,就走大模型 String answer = aiService.talk(question, medicalRec.getMedicalRecordAi()); // 记录大模型的流转记录 buildAiCirculationDetail(circulationList, answer, medicalRec); // 保存消息到记录表 saveQaRecord(process.getId(), medicalRec, 2, question, null, answer, circulationList); return answer; } QaSimilarityQuestionAnswer similarityResult = first.get(); // 如果阈值过低,也走大模型 double thresholdValue = Double.parseDouble(threshold); if (similarityResult.getMatchScore() < thresholdValue) { log.info("{}:匹配到的结果阈值过低,走大模型回答", similarityResult); circulationList.add(AskCirculationDetail.builder() .failInfo("相似度为:" + similarityResult.getMatchScore() + ",低于配置的阈值:" + threshold + ",走大模型") .similarityInfo(JSONUtil.toJsonStr(similarityAnswerList)) .build()); String answer = aiService.talk(question, medicalRec.getMedicalRecordAi()); // 记录流转记录 buildAiCirculationDetail(circulationList, answer, medicalRec); saveQaRecord(process.getId(), medicalRec, 2, question, null, answer, circulationList); return answer; } // 根据对应的标准问题,从标准问题表中找到标准问题 AskTemplateQuestionLibrary library = askTemplateQuestionLibraryService.getById(similarityResult.getLibraryQuestionId()); if (ObjectUtil.isEmpty(library)) { log.info("{}:未从问题库中找到答案,走大模型回答", similarityResult); circulationList.add(AskCirculationDetail.builder() // 问题库中未找到该问题 .failInfo("问题库中匹配到的问题为:" + similarityResult.getMatchQuestion() + ",但未找到该标准问题:" + similarityResult.getLibraryQuestionId() + ",走大模型") .similarityInfo(JSONUtil.toJsonStr(similarityAnswerList)) .build()); String answer = aiService.talk(question, medicalRec.getMedicalRecordAi()); // 记录流转记录 buildAiCirculationDetail(circulationList, answer, medicalRec); saveQaRecord(process.getId(), medicalRec, 2, question, null, answer, circulationList); return answer; } // 根据问题找这个病历配置的答案 AskPatientAnswer askPatientAnswer = askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, process.getMedicalRecId()) .eq(AskPatientAnswer::getLibraryQuestionId, library.getId()).last("limit 1").one(); // 如果问题的答案没有配置,还是走大模型的回答 if (ObjectUtil.isEmpty(askPatientAnswer)) { circulationList.add(AskCirculationDetail.builder() .failInfo("相似问匹配到了,但未找到对应的病历配置的回答,匹配到的标准问ID为:" + similarityResult.getLibraryQuestionId()) .similarityInfo(JSONUtil.toJsonStr(similarityAnswerList)) .build()); log.info("{}:病历配置,从AskPatientAnswer中未找到回答结果,走大模型", similarityResult.getLibraryQuestionId()); String answer = aiService.talk(question, medicalRec.getMedicalRecordAi()); // 记录流转记录 buildAiCirculationDetail(circulationList, answer, medicalRec); saveQaRecord(process.getId(), medicalRec, 2, question, null, answer, circulationList); return answer; } // 如果找到了,就走病历配置的内容回答 String patientAnswer = askPatientAnswer.getAnswer(); log.info("{}:找到了病历配置的回答语句:{},回答内容:{},走病历回答", similarityResult.getMatchQuestion(), askPatientAnswer.getId(), patientAnswer); circulationList.add(AskCirculationDetail.builder() .answer(patientAnswer) // 问题库中未找到该问题 .successInfo("相似问匹配到了,走相似问") .similarityInfo(JSONUtil.toJsonStr(similarityAnswerList)) .matchScore(similarityResult.getMatchScore()) .matchItem(Optional.ofNullable(commonDicService.getById(Long.parseLong(similarityResult.getDictId()))).orElse(new CommonDic()).getNameZhPath()) .matchSimilarityId(similarityResult.getMatchQuestionId()) .matchLibraryId(similarityResult.getLibraryQuestionId()) .matchQuestion(similarityResult.getMatchQuestion()) .successType(1) .build()); saveQaRecord(process.getId(), medicalRec, 1, question, similarityResult.getLibraryQuestionId(), patientAnswer, circulationList); return patientAnswer; } private void buildAiCirculationDetail(List detailList, String answer, MedicalRec medicalRec) { AskCirculationDetail newDetail = AskCirculationDetail.builder().answer(answer) .successInfo("走大模型进行匹配,获得答案,回复用户").aiMedicalContext(medicalRec.getMedicalRecordAi()) .successType(2).build(); detailList.add(newDetail); } private void saveQaRecord(String processId, MedicalRec medicalRec, Integer matchType, String question, String libraryId, String answer, List circulationList) { DiagnosisQaRecord record = new DiagnosisQaRecord(); record.setProcessId(processId); record.setMatchType(matchType); record.setQuestionLibraryId(libraryId); record.setQuestion(question); record.setAnswer(answer); record.setCreateUserId(UserUtil.getUser().getId()); record.insert(); AtomicInteger atomicInteger = new AtomicInteger(0); circulationList.forEach(e -> { e.setProcessId(processId); e.setMedicalId(medicalRec.getId()); e.setRecordId(record.getId()); e.setCirculationNo(atomicInteger.incrementAndGet()); e.setQuestion(question); } ); askCirculationDetailService.saveBatch(circulationList); } }