|
|
|
|
@ -0,0 +1,258 @@
|
|
|
|
|
package com.supervision.manage.service.impl;
|
|
|
|
|
|
|
|
|
|
import cn.hutool.core.bean.BeanUtil;
|
|
|
|
|
import cn.hutool.core.collection.CollUtil;
|
|
|
|
|
import cn.hutool.core.lang.Assert;
|
|
|
|
|
import cn.hutool.core.lang.UUID;
|
|
|
|
|
import cn.hutool.core.util.StrUtil;
|
|
|
|
|
import cn.hutool.json.JSONObject;
|
|
|
|
|
import com.supervision.config.RedisVectorProperties;
|
|
|
|
|
import com.supervision.config.RedisVectorStoreInit;
|
|
|
|
|
import com.supervision.config.VectorEmbeddingClient;
|
|
|
|
|
import com.supervision.domain.QaSimilarityQuestionAnswer;
|
|
|
|
|
import com.supervision.exception.BusinessException;
|
|
|
|
|
import com.supervision.manage.pojo.vo.AssessCustomQuestionReqVO;
|
|
|
|
|
import com.supervision.manage.pojo.vo.CustomQuestionMatchResVO;
|
|
|
|
|
import com.supervision.manage.pojo.vo.CustomQuestionSaveReqVO;
|
|
|
|
|
import com.supervision.manage.pojo.vo.SaveCustomQuestionResVO;
|
|
|
|
|
import com.supervision.manage.service.CustomQuestionService;
|
|
|
|
|
import com.supervision.model.AskTemplateQuestionLibrary;
|
|
|
|
|
import com.supervision.model.AskTemplateQuestionSimilarity;
|
|
|
|
|
import com.supervision.model.CommonDic;
|
|
|
|
|
import com.supervision.record.QuestionMetadata;
|
|
|
|
|
import com.supervision.service.AskTemplateQuestionLibraryService;
|
|
|
|
|
import com.supervision.service.AskTemplateQuestionSimilarityService;
|
|
|
|
|
import com.supervision.service.CommonDicService;
|
|
|
|
|
import com.supervision.util.AiChatUtil;
|
|
|
|
|
import com.supervision.util.SimilarityUtil;
|
|
|
|
|
import lombok.RequiredArgsConstructor;
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
import org.springframework.ai.document.Document;
|
|
|
|
|
import org.springframework.ai.vectorstore.RedisVectorStore;
|
|
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
import org.springframework.transaction.annotation.Transactional;
|
|
|
|
|
|
|
|
|
|
import java.util.*;
|
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
|
|
|
|
@Service
|
|
|
|
|
@RequiredArgsConstructor
|
|
|
|
|
@Slf4j
|
|
|
|
|
public class CustomQuestionServiceImpl implements CustomQuestionService {
|
|
|
|
|
|
|
|
|
|
private static final String prompt = """
|
|
|
|
|
根据用户输入的问题,生成相似的不同问法.要求最少生成20条,且保证和输入的问题相似
|
|
|
|
|
问题:{customQuestion}
|
|
|
|
|
回复格式:{"similarList":["xx1","xx2"]}
|
|
|
|
|
""";
|
|
|
|
|
private final AskTemplateQuestionLibraryService askTemplateQuestionLibraryService;
|
|
|
|
|
|
|
|
|
|
private final CommonDicService commonDicService;
|
|
|
|
|
|
|
|
|
|
private final RedisVectorProperties redisVectorProperties;
|
|
|
|
|
|
|
|
|
|
private final VectorEmbeddingClient vectorEmbeddingClient;
|
|
|
|
|
|
|
|
|
|
private final AskTemplateQuestionSimilarityService askTemplateQuestionSimilarityService;
|
|
|
|
|
|
|
|
|
|
private final RedisVectorStore redisVectorStore;
|
|
|
|
|
@Value("${customQuestionThreshold:0.6}")
|
|
|
|
|
private String customQuestionThreshold;
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public CustomQuestionMatchResVO assessCustomQuestion(AssessCustomQuestionReqVO reqVO) {
|
|
|
|
|
CustomQuestionMatchResVO resVO = new CustomQuestionMatchResVO();
|
|
|
|
|
// 首先拿问题去向量库里面进行匹配,找到TOP的数据
|
|
|
|
|
Assert.notBlank(reqVO.getCustomQuestion(), "问诊问题不能为空");
|
|
|
|
|
if (CollUtil.isEmpty(reqVO.getSelectedQuestionLibraryIdList())) {
|
|
|
|
|
// 如果没有选择过任何问题,则直接允许添加
|
|
|
|
|
resVO.setAccessCustom(true);
|
|
|
|
|
return resVO;
|
|
|
|
|
}
|
|
|
|
|
// 校验相似度
|
|
|
|
|
List<SimilarLibraryQuestionResVO> similarList = checkCustomQuestionSimilar(reqVO.getCustomQuestion(), reqVO.getSelectedQuestionLibraryIdList());
|
|
|
|
|
if (CollUtil.isEmpty(similarList)) {
|
|
|
|
|
// 如果走这里,说明相似度比较低,可以允许
|
|
|
|
|
resVO.setAccessCustom(true);
|
|
|
|
|
} else {
|
|
|
|
|
resVO.setAccessCustom(false);
|
|
|
|
|
resVO.setSimilarLibraryQuestionList(similarList);
|
|
|
|
|
}
|
|
|
|
|
return resVO;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private List<SimilarLibraryQuestionResVO> checkCustomQuestionSimilar(String customQuestion, List<String> selectedQuestionLibraryIdList) {
|
|
|
|
|
// 只和已经选择的问题库进行比较
|
|
|
|
|
List<QaSimilarityQuestionAnswer> qaSimilarityQuestionAnswers = SimilarityUtil.talkRedisVectorWithScore(customQuestion, selectedQuestionLibraryIdList);
|
|
|
|
|
// 找到大于阈值的
|
|
|
|
|
List<QaSimilarityQuestionAnswer> overThresholdList = qaSimilarityQuestionAnswers.stream()
|
|
|
|
|
.filter(e -> e.getMatchScore() >= Double.parseDouble(customQuestionThreshold)).toList();
|
|
|
|
|
List<SimilarLibraryQuestionResVO> similarList = new ArrayList<>();
|
|
|
|
|
if (CollUtil.isNotEmpty(overThresholdList)) {
|
|
|
|
|
// 如果不为空,说明有相似度比较高的,这时,获取他们的标准库ID
|
|
|
|
|
LinkedHashSet<Object> overLibraryIdSet = new LinkedHashSet<>();
|
|
|
|
|
for (QaSimilarityQuestionAnswer similarityQuestionAnswer : overThresholdList) {
|
|
|
|
|
overLibraryIdSet.add(similarityQuestionAnswer.getLibraryQuestionId());
|
|
|
|
|
}
|
|
|
|
|
// 然后获取标准库的相似问题以及关联的字典
|
|
|
|
|
List<AskTemplateQuestionLibrary> libraryList = askTemplateQuestionLibraryService.lambdaQuery()
|
|
|
|
|
.in(AskTemplateQuestionLibrary::getId, overLibraryIdSet).list();
|
|
|
|
|
List<Long> dictIdList = libraryList.stream().map(AskTemplateQuestionLibrary::getDictId).toList();
|
|
|
|
|
List<CommonDic> dictList = commonDicService.lambdaQuery().in(CommonDic::getId, dictIdList).list();
|
|
|
|
|
Map<Long, String> dictMap = dictList.stream().collect(Collectors.toMap(CommonDic::getId, CommonDic::getNameZhPath));
|
|
|
|
|
|
|
|
|
|
for (AskTemplateQuestionLibrary library : libraryList) {
|
|
|
|
|
SimilarLibraryQuestionResVO similarNode = new SimilarLibraryQuestionResVO();
|
|
|
|
|
similarNode.setLibraryQuestion(library.getStandardQuestion());
|
|
|
|
|
similarNode.setDictPath(dictMap.get(library.getDictId()));
|
|
|
|
|
similarList.add(similarNode);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return similarList;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public List<String> generateSimilarQuestion(AssessCustomQuestionReqVO reqVO) {
|
|
|
|
|
log.info("开始调用大模型生成相似问题");
|
|
|
|
|
Optional<JSONObject> chat = AiChatUtil.chat(StrUtil.format(prompt, Map.of("customQuestion", reqVO.getCustomQuestion())));
|
|
|
|
|
// 最终相似的列表
|
|
|
|
|
log.info("调用大模型生成相似问题结束");
|
|
|
|
|
if (chat.isPresent()) {
|
|
|
|
|
List<String> similarList = chat.get().getBeanList("similarList", String.class);
|
|
|
|
|
return checkSimilarQuestion(reqVO.getCustomQuestion(), similarList, reqVO.getSelectedQuestionLibraryIdList());
|
|
|
|
|
}
|
|
|
|
|
return new ArrayList<>();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 潜在bug:
|
|
|
|
|
* 这里可能有一个问题,如果先添加的自定义的相似问,再去选择一个标准问
|
|
|
|
|
* 这时,我们是不对标准问下面的相似问,再和自定义问题以及对应的相似问进行一遍相似度校验的
|
|
|
|
|
* 这时,如果自定义的相似问和标准问的相似问如果相似度很高,就会出现答非所问的情况
|
|
|
|
|
* 2024.06.18 15:24和产品反馈了这个问题,答案是,先不管这个问题,等后面实际使用中出现了,再去想怎么解决
|
|
|
|
|
* weixin聊天记录:避免限制配置，我们按医生提供的问题配置完后会去实际测试问诊，如果有答非所问的情况，会再人工删除
|
|
|
|
|
*
|
|
|
|
|
* @param customQuestion 用户自定义的标注问题
|
|
|
|
|
* @param similarQuestionList 大模型或用户手动填写的相似问
|
|
|
|
|
* @param selectedQuestionList 用户已选择的标准问
|
|
|
|
|
* @return 相似问
|
|
|
|
|
*/
|
|
|
|
|
private List<String> checkSimilarQuestion(String customQuestion, List<String> similarQuestionList, List<String> selectedQuestionList) {
|
|
|
|
|
List<String> similarResultList = new ArrayList<>();
|
|
|
|
|
// 定义一个新的向量库
|
|
|
|
|
RedisVectorProperties newRedisVectorProperties = BeanUtil.copyProperties(redisVectorProperties, RedisVectorProperties.class);
|
|
|
|
|
// 设置一个新的前缀和indexName
|
|
|
|
|
String uuid = UUID.fastUUID().toString();
|
|
|
|
|
newRedisVectorProperties.setPrefix("customQuestion:" + uuid + ":");
|
|
|
|
|
String indexName = StrUtil.format("{}-{}", "customQuestion-index", uuid);
|
|
|
|
|
newRedisVectorProperties.setIndexName(indexName);
|
|
|
|
|
RedisVectorStore tempRedisVectorStore = RedisVectorStoreInit.redisVectorStoreInit(vectorEmbeddingClient, newRedisVectorProperties);
|
|
|
|
|
tempRedisVectorStore.afterPropertiesSet();
|
|
|
|
|
// 先把用户自定义的标准问题加载进去,然后遍历相似问,去计算相似问和这个自定义的标准问题间的相似度.相似度高才允许被保留
|
|
|
|
|
ArrayList<String> tempVectorList = new ArrayList<>();
|
|
|
|
|
List<Document> documents = new ArrayList<>();
|
|
|
|
|
Document document = new Document(customQuestion);
|
|
|
|
|
documents.add(document);
|
|
|
|
|
tempVectorList.add(document.getId());
|
|
|
|
|
tempRedisVectorStore.add(documents);
|
|
|
|
|
log.info("生成临时库以存储临时的标准问完成");
|
|
|
|
|
try {
|
|
|
|
|
// 首先对生成的相似问和提交的标准问进行相似度比较,如果分数比较低,就忽略
|
|
|
|
|
for (String similarQuestion : similarQuestionList) {
|
|
|
|
|
log.info("正在计算相似度:{}", similarQuestion);
|
|
|
|
|
List<Document> similaritySearchList = tempRedisVectorStore.similaritySearch(similarQuestion);
|
|
|
|
|
if (CollUtil.isNotEmpty(similaritySearchList)) {
|
|
|
|
|
// 如果相似度比较高,就添加进去
|
|
|
|
|
Double similarityScore = SimilarityUtil.computeScore(similaritySearchList.get(0));
|
|
|
|
|
log.info("问题:{}和自定义问题的相似度为:{}", similarQuestion, similarityScore);
|
|
|
|
|
if (similarityScore >= 0.6) {
|
|
|
|
|
// 如果没有选择任何问题,那么就直接过
|
|
|
|
|
if (CollUtil.isNotEmpty(selectedQuestionList)) {
|
|
|
|
|
// 再过一遍已选择的问题,如果已选择的问题中存在相似度较高的,就不允许添加进来
|
|
|
|
|
List<QaSimilarityQuestionAnswer> qaSimilarityQuestionAnswers = SimilarityUtil.talkRedisVectorWithScore(similarQuestion, selectedQuestionList);
|
|
|
|
|
if (CollUtil.isNotEmpty(qaSimilarityQuestionAnswers)) {
|
|
|
|
|
// 如果标准库的数据比较低,才允许添加进来
|
|
|
|
|
Double qaMatchScore = qaSimilarityQuestionAnswers.get(0).getMatchScore();
|
|
|
|
|
log.info("问题:{}和标准问题库的相似度为:{},匹配度最高的标准问题库问题为:{}", similarQuestion, qaMatchScore, qaSimilarityQuestionAnswers.get(0).getMatchQuestion());
|
|
|
|
|
if (qaMatchScore <= 0.6) {
|
|
|
|
|
similarResultList.add(similarQuestion);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
similarResultList.add(similarQuestion);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
log.info("计算自定义的标准问题的和相似问的相似度出现问题");
|
|
|
|
|
throw e;
|
|
|
|
|
} finally {
|
|
|
|
|
// 最终要进行删除
|
|
|
|
|
tempRedisVectorStore.delete(tempVectorList);
|
|
|
|
|
tempRedisVectorStore.getJedis().ftDropIndex(indexName);
|
|
|
|
|
}
|
|
|
|
|
return similarResultList;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
@Transactional(rollbackFor = Exception.class)
|
|
|
|
|
public SaveCustomQuestionResVO saveCustomQuestion(CustomQuestionSaveReqVO reqVO) {
|
|
|
|
|
// 首先对标准问进行校验,校验相似度比较低才能允许被提交
|
|
|
|
|
List<SimilarLibraryQuestionResVO> similarLibraryQuestionResVOS = checkCustomQuestionSimilar(reqVO.getCustomQuestion(), reqVO.getSelectedQuestionLibraryIdList());
|
|
|
|
|
if (CollUtil.isNotEmpty(similarLibraryQuestionResVOS)) {
|
|
|
|
|
throw new BusinessException("相似度较高，已存在相似问诊问题，不支持添加，若要添加可联系运营人员!");
|
|
|
|
|
}
|
|
|
|
|
List<String> similarQuestionList = checkSimilarQuestion(reqVO.getCustomQuestion(), reqVO.getSimilarQuestionList(), reqVO.getSelectedQuestionLibraryIdList());
|
|
|
|
|
// 校验,这时要校验哪些问题没有被添加进来,如果没有被添加进来,说明相似度较高,这时前端应该给于提示
|
|
|
|
|
List<String> errorSimilarQuestionList = new ArrayList<>();
|
|
|
|
|
for (String pageSimilarQuestion : reqVO.getSimilarQuestionList()) {
|
|
|
|
|
// 如果不包含,说明相似度较高
|
|
|
|
|
if (!similarQuestionList.contains(pageSimilarQuestion)) {
|
|
|
|
|
errorSimilarQuestionList.add(pageSimilarQuestion);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
SaveCustomQuestionResVO saveCustomQuestionResVO = new SaveCustomQuestionResVO();
|
|
|
|
|
if (CollUtil.isNotEmpty(errorSimilarQuestionList)) {
|
|
|
|
|
saveCustomQuestionResVO.setSaveResult(false);
|
|
|
|
|
saveCustomQuestionResVO.setErrorSimilarQuestionList(errorSimilarQuestionList);
|
|
|
|
|
return saveCustomQuestionResVO;
|
|
|
|
|
}
|
|
|
|
|
// 如果为空,说明符合要求,这时就进行保存
|
|
|
|
|
AskTemplateQuestionLibrary askTemplateQuestionLibrary = new AskTemplateQuestionLibrary();
|
|
|
|
|
askTemplateQuestionLibrary.setDictId(reqVO.getDictId());
|
|
|
|
|
askTemplateQuestionLibrary.setStandardQuestion(reqVO.getCustomQuestion());
|
|
|
|
|
askTemplateQuestionLibrary.setType(2);
|
|
|
|
|
askTemplateQuestionLibraryService.save(askTemplateQuestionLibrary);
|
|
|
|
|
// 保存相似问
|
|
|
|
|
for (String similarQuestion : similarQuestionList) {
|
|
|
|
|
AskTemplateQuestionSimilarity askTemplateQuestionSimilarity = new AskTemplateQuestionSimilarity();
|
|
|
|
|
askTemplateQuestionSimilarity.setLibraryId(askTemplateQuestionLibrary.getId());
|
|
|
|
|
askTemplateQuestionSimilarity.setSimilarityQuestion(similarQuestion);
|
|
|
|
|
askTemplateQuestionSimilarityService.save(askTemplateQuestionSimilarity);
|
|
|
|
|
}
|
|
|
|
|
// 这时,需要将数据刷到向量库中
|
|
|
|
|
String description = askTemplateQuestionLibrary.getStandardQuestion();
|
|
|
|
|
Map<String, Object> questionVector = new QuestionMetadata("3",
|
|
|
|
|
askTemplateQuestionLibrary.getId(), askTemplateQuestionLibrary.getId(),
|
|
|
|
|
askTemplateQuestionLibrary.getDictId()).toMap();
|
|
|
|
|
redisVectorStore.add(List.of(new Document(askTemplateQuestionLibrary.getId(), description, questionVector)));
|
|
|
|
|
|
|
|
|
|
List<AskTemplateQuestionSimilarity> similarityList = askTemplateQuestionSimilarityService.lambdaQuery()
|
|
|
|
|
.eq(AskTemplateQuestionSimilarity::getLibraryId, askTemplateQuestionLibrary.getId()).list();
|
|
|
|
|
for (AskTemplateQuestionSimilarity askTemplateQuestionSimilarity : similarityList) {
|
|
|
|
|
Map<String, Object> questionVectorNest = new QuestionMetadata("2",
|
|
|
|
|
askTemplateQuestionSimilarity.getLibraryId(), askTemplateQuestionSimilarity.getId(),
|
|
|
|
|
askTemplateQuestionLibrary.getDictId()).toMap();
|
|
|
|
|
redisVectorStore.add(List.of(new Document(askTemplateQuestionSimilarity.getId(),
|
|
|
|
|
askTemplateQuestionSimilarity.getSimilarityQuestion(),
|
|
|
|
|
questionVectorNest)));
|
|
|
|
|
}
|
|
|
|
|
saveCustomQuestionResVO.setSaveResult(true);
|
|
|
|
|
return saveCustomQuestionResVO;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}
|
liu
10 months ago