|
|
|
|
@ -1,30 +1,18 @@
|
|
|
|
|
package com.supervision.service.impl;
|
|
|
|
|
|
|
|
|
|
import cn.hutool.core.codec.Base64Encoder;
|
|
|
|
|
import cn.hutool.core.collection.CollUtil;
|
|
|
|
|
import cn.hutool.core.io.IoUtil;
|
|
|
|
|
import cn.hutool.core.lang.Assert;
|
|
|
|
|
import cn.hutool.core.util.ObjectUtil;
|
|
|
|
|
import cn.hutool.core.util.StrUtil;
|
|
|
|
|
import cn.hutool.crypto.digest.MD5;
|
|
|
|
|
import cn.hutool.json.JSONUtil;
|
|
|
|
|
import com.supervision.domain.GlobalResult;
|
|
|
|
|
import com.supervision.exception.BusinessException;
|
|
|
|
|
import com.supervision.feign.AskQaSimilarityFeignClient;
|
|
|
|
|
import com.supervision.feign.RasaManageFeignClient;
|
|
|
|
|
import com.supervision.model.Process;
|
|
|
|
|
import com.supervision.model.*;
|
|
|
|
|
import com.supervision.pojo.qaSimilarity.QaSimilarityQuestion;
|
|
|
|
|
import com.supervision.pojo.qaSimilarity.QaSimilarityQuestionAnswer;
|
|
|
|
|
import com.supervision.pojo.vo.TalkResultResVO;
|
|
|
|
|
import com.supervision.pojo.vo.TalkVideoReqVO;
|
|
|
|
|
import com.supervision.pojo.vo.TalkVideoTtsResultResVO;
|
|
|
|
|
import com.supervision.service.*;
|
|
|
|
|
import com.supervision.util.AsrUtil;
|
|
|
|
|
import com.supervision.util.MinioUtil;
|
|
|
|
|
import com.supervision.util.TtsUtil;
|
|
|
|
|
import com.supervision.util.UserUtil;
|
|
|
|
|
import com.supervision.vo.rasa.RasaTalkVo;
|
|
|
|
|
import lombok.RequiredArgsConstructor;
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
import org.springframework.ai.document.Document;
|
|
|
|
|
@ -33,7 +21,6 @@ import org.springframework.beans.factory.annotation.Value;
|
|
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
import org.springframework.web.multipart.MultipartFile;
|
|
|
|
|
|
|
|
|
|
import java.io.InputStream;
|
|
|
|
|
import java.util.Comparator;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.Optional;
|
|
|
|
|
@ -49,18 +36,6 @@ public class AskServiceImpl implements AskService {
|
|
|
|
|
|
|
|
|
|
private final AskPatientAnswerService askPatientAnswerService;
|
|
|
|
|
|
|
|
|
|
private final ConfigPhysicalToolService configPhysicalToolService;
|
|
|
|
|
|
|
|
|
|
private final ConfigAncillaryItemService configAncillaryItemService;
|
|
|
|
|
|
|
|
|
|
private final FileResourceService fileResourceService;
|
|
|
|
|
|
|
|
|
|
private final RasaManageFeignClient rasaManageFeignClient;
|
|
|
|
|
|
|
|
|
|
private final AskQaSimilarityFeignClient askQaSimilarityFeignClient;
|
|
|
|
|
|
|
|
|
|
private final CommonDicService commonDicService;
|
|
|
|
|
|
|
|
|
|
private final AiService aiService;
|
|
|
|
|
|
|
|
|
|
private final MedicalRecService medicalRecService;
|
|
|
|
|
@ -68,8 +43,8 @@ public class AskServiceImpl implements AskService {
|
|
|
|
|
private final DiagnosisAiRecordService diagnosisAiRecordService;
|
|
|
|
|
|
|
|
|
|
private final RedisVectorStore redisVectorStore;
|
|
|
|
|
@Value("${qaSimilarityThreshold:0.7}")
|
|
|
|
|
private String qaSimilarityThreshold;
|
|
|
|
|
@Value("${threshold:0.7}")
|
|
|
|
|
private String threshold;
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public String receiveVoiceFile(MultipartFile file) {
|
|
|
|
|
@ -104,139 +79,6 @@ public class AskServiceImpl implements AskService {
|
|
|
|
|
record.insert();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private void setActionRelation(TalkResultResVO talkResultResVO) {
|
|
|
|
|
Integer type = talkResultResVO.getType();
|
|
|
|
|
String actionId = talkResultResVO.getActionId();
|
|
|
|
|
if (StrUtil.isEmpty(actionId) || null == type) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 体格检查
|
|
|
|
|
if (type.equals(2)) {
|
|
|
|
|
ConfigPhysicalTool configPhysicalTool = configPhysicalToolService.getById(actionId);
|
|
|
|
|
if (null != configPhysicalTool) {
|
|
|
|
|
talkResultResVO.setItemName(configPhysicalTool.getToolName());
|
|
|
|
|
talkResultResVO.setItemImage(configPhysicalTool.getIconBase64());
|
|
|
|
|
talkResultResVO.setRequireLocation(configPhysicalTool.getRequireLocation());
|
|
|
|
|
// 生成树的时候,没有ID字段,所以用名称生成md5作为ID
|
|
|
|
|
talkResultResVO.setActionType(new MD5().digestHex16(configPhysicalTool.getType()));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 辅助检查
|
|
|
|
|
if (type.equals(3)) {
|
|
|
|
|
ConfigAncillaryItem configAncillaryItem = configAncillaryItemService.getById(actionId);
|
|
|
|
|
if (null != configAncillaryItem) {
|
|
|
|
|
talkResultResVO.setItemName(configAncillaryItem.getItemName());
|
|
|
|
|
// 生成树的时候,没有ID字段,所以用名称生成md5作为ID
|
|
|
|
|
talkResultResVO.setActionType(new MD5().digestHex16(configAncillaryItem.getType()));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public TalkResultResVO talkByVideo(TalkVideoReqVO talkReqVO) {
|
|
|
|
|
|
|
|
|
|
// 根据processId找到对应的病人
|
|
|
|
|
Process process = Optional.ofNullable(processService.getById(talkReqVO.getProcessId())).orElseThrow(() -> new BusinessException("未找到诊疗进程"));
|
|
|
|
|
String talkResult = talkQaSimilarity(talkReqVO.getText(), UserUtil.getUser().getId());
|
|
|
|
|
// 调用rasa获取文字内容
|
|
|
|
|
//String rasaResult = talkRasa(talkReqVO.getText(), UserUtil.getUser().getId());
|
|
|
|
|
log.info("调用对话的回复是:{}", talkResult);
|
|
|
|
|
TalkResultResVO talkResultResVO = new TalkResultResVO();
|
|
|
|
|
// 如果rasa没有识别出来,则返回默认值
|
|
|
|
|
if (StrUtil.isBlank(talkResult)) {
|
|
|
|
|
AskPatientAnswer medicalRecErrorAnswer = getMedicalRecErrorAnswer(process.getMedicalRecId());
|
|
|
|
|
talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(medicalRecErrorAnswer.getAnswerResourceId()));
|
|
|
|
|
talkResultResVO.setAnswerMessage(medicalRecErrorAnswer.getAnswer());
|
|
|
|
|
saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, medicalRecErrorAnswer.getAnswer());
|
|
|
|
|
talkResultResVO.setType(1);
|
|
|
|
|
return talkResultResVO;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 这里校验,rasa回复的结果是不是action
|
|
|
|
|
// 这里设置的模板,对于action的动作全部是用ancillary_ | tool_进行标记,详情看生成rasa的yml的代码:RasaServiceImpl.generateDomain
|
|
|
|
|
// ancillary_ | tool_
|
|
|
|
|
if (talkResult.startsWith("ancillary_") || talkResult.startsWith("tool_")) {
|
|
|
|
|
log.info("呼出语句:{}", talkResult);
|
|
|
|
|
List<String> actionList = StrUtil.split(talkResult, '_');
|
|
|
|
|
if (actionList.size() > 1) {
|
|
|
|
|
// 在这里设置为动作
|
|
|
|
|
talkResultResVO.setActionId(actionList.get(1));
|
|
|
|
|
talkResultResVO.setType("ancillary".equals(actionList.get(0)) ? 3 : 2);
|
|
|
|
|
setActionRelation(talkResultResVO);
|
|
|
|
|
return talkResultResVO;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
AskTemplateQuestionLibrary library = askTemplateQuestionLibraryService.getById(talkResult);
|
|
|
|
|
if (ObjectUtil.isEmpty(library)) {
|
|
|
|
|
log.info("{}:未从问题库中找到,回答未识别语句", talkResult);
|
|
|
|
|
AskPatientAnswer medicalRecErrorAnswer = getMedicalRecErrorAnswer(process.getMedicalRecId());
|
|
|
|
|
talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(medicalRecErrorAnswer.getAnswerResourceId()));
|
|
|
|
|
talkResultResVO.setAnswerMessage(medicalRecErrorAnswer.getAnswer());
|
|
|
|
|
saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), null, medicalRecErrorAnswer.getAnswer());
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
|
|
AskPatientAnswer askPatientAnswer = askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, process.getMedicalRecId())
|
|
|
|
|
.eq(AskPatientAnswer::getLibraryQuestionId, library.getId()).last("limit 1").one();
|
|
|
|
|
|
|
|
|
|
if (ObjectUtil.isNotEmpty(askPatientAnswer) && StrUtil.isNotEmpty(askPatientAnswer.getAnswerResourceId())) {
|
|
|
|
|
String resText = askPatientAnswer.getAnswer();
|
|
|
|
|
log.info("{}:找到了病历配置的回答语句:{},回答内容:{}", talkResult, askPatientAnswer.getId(), resText);
|
|
|
|
|
talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(askPatientAnswer.getAnswerResourceId()));
|
|
|
|
|
talkResultResVO.setAnswerMessage(askPatientAnswer.getAnswer());
|
|
|
|
|
// 保存记录
|
|
|
|
|
saveQaRecord(talkReqVO.getProcessId(), "patient", askPatientAnswer.getId(), talkReqVO.getText(), library, resText);
|
|
|
|
|
} else {
|
|
|
|
|
// 临时解决方案,回答没有,在问题库中内置这个回答
|
|
|
|
|
log.info("{}:病历配置,从AskPatientAnswer中未找到回答结果,回复 默认回复", talkResult);
|
|
|
|
|
AskPatientAnswer medicalRecDefaultAnswer = getMedicalRecDefaultAnswer(process.getMedicalRecId());
|
|
|
|
|
talkResultResVO.setVideoBase64(getAnswerVideoBase64OrDefault(medicalRecDefaultAnswer.getAnswerResourceId()));
|
|
|
|
|
talkResultResVO.setAnswerMessage(medicalRecDefaultAnswer.getAnswer());
|
|
|
|
|
saveQaRecord(talkReqVO.getProcessId(), "default", null, talkReqVO.getText(), library, medicalRecDefaultAnswer.getAnswer());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
talkResultResVO.setType(1);
|
|
|
|
|
return talkResultResVO;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* rasa接口,不好用,现在已废弃
|
|
|
|
|
*/
|
|
|
|
|
@Deprecated
|
|
|
|
|
public String talkRasa(String question, String sessionId) {
|
|
|
|
|
RasaTalkVo rasaTalkVo = new RasaTalkVo();
|
|
|
|
|
rasaTalkVo.setQuestion(question);
|
|
|
|
|
rasaTalkVo.setSessionId(sessionId);
|
|
|
|
|
// 默认为1
|
|
|
|
|
rasaTalkVo.setModelId("1");
|
|
|
|
|
try {
|
|
|
|
|
GlobalResult<List<String>> talk = rasaManageFeignClient.talk(rasaTalkVo);
|
|
|
|
|
log.info("调用rasa对话返回结果:{}", talk);
|
|
|
|
|
if (talk.getCode() != 200 || CollUtil.isEmpty(talk.getData())) {
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
return CollUtil.getFirst(talk.getData());
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
log.error("talkRasa error ", e);
|
|
|
|
|
}
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public String talkQaSimilarity(String question, String sessionId) {
|
|
|
|
|
log.info("开始调用talkQaSimilarity,问题:{}", question);
|
|
|
|
|
try {
|
|
|
|
|
GlobalResult<List<QaSimilarityQuestionAnswer>> result = askQaSimilarityFeignClient.askQuestionSimilarityAnswer(new QaSimilarityQuestion(question));
|
|
|
|
|
log.info("调用talkQaSimilarity结束,问题:{},返回结果:{}", question, JSONUtil.toJsonStr(result));
|
|
|
|
|
return CollUtil.getFirst(result.getData()).getMatchQuestionCode();
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
log.error("调用talkQaSimilarity error ", e);
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public QaSimilarityQuestionAnswer talkRedisVectorWithScore(String question) {
|
|
|
|
|
log.info("开始调用talkQaSimilarity,问题:{}", question);
|
|
|
|
|
try {
|
|
|
|
|
@ -258,47 +100,6 @@ public class AskServiceImpl implements AskService {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private AskPatientAnswer getMedicalRecErrorAnswer(String medicalRecId) {
|
|
|
|
|
//Optional.ofNullable(medicalRecErrorAnswer).orElseGet(() ->new AskPatientAnswer()).getAnswer()
|
|
|
|
|
Assert.notEmpty(medicalRecId, "病历id不能为空");
|
|
|
|
|
CommonDic commonDic = commonDicService.lambdaQuery().eq(CommonDic::getGroupCode, "AQT").eq(CommonDic::getCode, "system_error").one();
|
|
|
|
|
Assert.notNull(commonDic, "字典未配置系统内置异常识别场景");
|
|
|
|
|
|
|
|
|
|
AskTemplateQuestionLibrary askTemplateQuestionLibrary = askTemplateQuestionLibraryService.lambdaQuery().eq(AskTemplateQuestionLibrary::getDictId, commonDic.getId()).one();
|
|
|
|
|
Assert.notNull(askTemplateQuestionLibrary, "知识库未配置系统内置异常识别场景");
|
|
|
|
|
|
|
|
|
|
return askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, medicalRecId).eq(AskPatientAnswer::getLibraryQuestionId, askTemplateQuestionLibrary.getId()).one();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private AskPatientAnswer getMedicalRecDefaultAnswer(String medicalRecId) {
|
|
|
|
|
//Optional.ofNullable(medicalRecErrorAnswer).orElseGet(() ->new AskPatientAnswer()).getAnswer()
|
|
|
|
|
Assert.notEmpty(medicalRecId, "病历id不能为空");
|
|
|
|
|
CommonDic commonDic = commonDicService.lambdaQuery().eq(CommonDic::getGroupCode, "AQT").eq(CommonDic::getCode, "system_default").one();
|
|
|
|
|
Assert.notNull(commonDic, "字典未配置默认回答场景");
|
|
|
|
|
|
|
|
|
|
AskTemplateQuestionLibrary askTemplateQuestionLibrary = askTemplateQuestionLibraryService.lambdaQuery().eq(AskTemplateQuestionLibrary::getDictId, commonDic.getId()).one();
|
|
|
|
|
Assert.notNull(askTemplateQuestionLibrary, "知识库未配置系统内置异常识别场景");
|
|
|
|
|
|
|
|
|
|
return askPatientAnswerService.lambdaQuery().eq(AskPatientAnswer::getMedicalId, medicalRecId).eq(AskPatientAnswer::getLibraryQuestionId, askTemplateQuestionLibrary.getId()).one();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 获取应答视频的base64位编码
|
|
|
|
|
*
|
|
|
|
|
* @param fileResourceId 应答id
|
|
|
|
|
* @return 应答视频base64位编码
|
|
|
|
|
*/
|
|
|
|
|
private String getAnswerVideoBase64OrDefault(String fileResourceId) {
|
|
|
|
|
FileResource fileResource = fileResourceService.getById(fileResourceId);
|
|
|
|
|
try (InputStream inputStream = MinioUtil.download(fileResource.getMinioId())) {
|
|
|
|
|
return Base64Encoder.encode(IoUtil.readBytes(inputStream));
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
log.error("获取视频失败", e);
|
|
|
|
|
throw new BusinessException("未找到回复视频");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 使用无声视频+语音转文字的形式来做
|
|
|
|
|
@ -320,8 +121,8 @@ public class AskServiceImpl implements AskService {
|
|
|
|
|
saveAiRecord(process.getId(), talkReqVO.getText(), talkVideoTtsResultResVO.getAnswerMessage());
|
|
|
|
|
} else {
|
|
|
|
|
// 如果阈值过低,也走大模型
|
|
|
|
|
double threshold = Double.parseDouble(qaSimilarityThreshold);
|
|
|
|
|
if (qaSimilarityQuestionAnswer.getMatchScore() < threshold) {
|
|
|
|
|
double thresholdValue = Double.parseDouble(threshold);
|
|
|
|
|
if (qaSimilarityQuestionAnswer.getMatchScore() < thresholdValue) {
|
|
|
|
|
log.info("{}:匹配到的结果阈值过低,走大模型回答", qaSimilarityQuestionAnswer);
|
|
|
|
|
String talk = aiService.talk(talkReqVO.getText(), medicalRec.getMedicalRecordAi());
|
|
|
|
|
talkVideoTtsResultResVO.setAnswerMessage(talk);
|
|
|
|
|
|
liu
11 months ago