interro_robot/src/main/java/com/supervision/service/impl/RobotTalkServiceImpl.java

522 lines
24 KiB
Java

package com.supervision.service.impl;
import cn.hutool.core.codec.Base64;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.convert.Convert;
import cn.hutool.core.date.TimeInterval;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.supervision.domain.*;
import com.supervision.dto.paddlespeech.res.TtsResultDTO;
import com.supervision.dto.robot.*;
import com.supervision.dto.matchTool.ExtractInformationDTO;
import com.supervision.dto.matchTool.MatchQuestionAnswerDTO;
import com.supervision.dto.QueryProcessDTO;
import com.supervision.mapper.RobotDataMapper;
import com.supervision.service.*;
import com.supervision.util.UserUtil;
import com.supervision.vo.robot.ArchivesReqVo;
import com.supervision.vo.robot.CommonDialogVo;
import com.supervision.vo.talk.RobotTalkReq;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Slf4j
@Service
@RequiredArgsConstructor
public class RobotTalkServiceImpl implements RobotTalkService {
private final QueryTemplateProcessor queryTemplateProcessor;
private final MatchToolService matchToolService;
private final VoiceService voiceService;
private final IrRobotConfigService irRobotConfigService;
private final IrSessionHistoryService sessionService;
private final IrSessionParamService irSessionParamService;
private final IrVoiceService irVoiceService;
private final IrFileService irFileService;
@Resource
private final RobotDataMapper robotDataMapper;
@Override
@Transactional(rollbackFor = Exception.class)
public RobotTalkDTO textTalk2Robot(RobotTalkReq robotTalkReq) {
String sessionId = robotTalkReq.getSessionId();
Assert.notEmpty(sessionId, "sessionId不能为空");
TimeInterval timeInterval = new TimeInterval();
timeInterval.start("all");
log.info("textTalk2Robot:开始问答,参数:{}",JSONUtil.toJsonStr(robotTalkReq));
RobotTalkDTO robotTalkDTO = RobotTalkDTO.builder()
.sessionId(sessionId).doNext(true)
.askInfo(AskInfo.builder().contentType(1).message(robotTalkReq.getMessage()).build())
.answerInfo(AnswerInfo.builder().contentType(robotTalkReq.getAnswerType()).build())
.build();
// 获取机器人配置
IrRobotConfig config = irRobotConfigService.lambdaQuery().one();
// 确认弹框信息
if (StrUtil.isNotEmpty(robotTalkReq.getAskId())){
IrSessionHistory sessionHistory = sessionService.lambdaQuery().eq(IrSessionHistory::getId, robotTalkReq.getAskId()).one();
if (Objects.nonNull(sessionHistory)){
robotTalkReq.setMessage(sessionHistory.getUserQuestion());
robotTalkReq.setConfirmFlag(true);
robotTalkDTO.getAskInfo().setAskId(robotTalkReq.getAskId());
}
}
// 匹配问题意图
MatchQuestionAnswerDTO matchQuestionAnswerDTO = matchQuestionAnswer(robotTalkReq.getMessage());
setSuspectInfoAndNextFlag(robotTalkReq, robotTalkDTO, matchQuestionAnswerDTO,config);
if (Objects.isNull(matchQuestionAnswerDTO) && robotTalkDTO.isDoNext()){
// 未匹配到查询意图,设置默认错误语
log.info("问题:{}未匹配到意图",robotTalkReq.getMessage());
robotTalkDTO.getAnswerInfo().setMessage(config.getErrorLanguage());
robotTalkDTO.setDoNext(false);
}
if (robotTalkDTO.isDoNext()){
QueryProcessDTO matchAnswer = queryMatch2Answer(sessionId, matchQuestionAnswerDTO, robotTalkDTO, robotTalkReq.getTitleContent());
if (Integer.valueOf(0).equals(matchAnswer.getState()) &&
Integer.valueOf(3).equals(matchAnswer.getContentType())
&& matchAnswer.getByteContent() != null){
// 查询结果类型为字节数组
robotTalkDTO.getAnswerInfo().setContentType(3);
robotTalkDTO.getAnswerInfo().setAnswerByte(matchAnswer.getByteContent());
}else{
robotTalkDTO.getAnswerInfo().setAnswerFile(matchAnswer.getFileContent());
robotTalkDTO.getAnswerInfo().setMessage(decideAnswer(matchAnswer, config));
if (matchAnswer.getState() == 1){
if (!Boolean.TRUE.equals(robotTalkReq.isConfirmFlag())){
robotTalkDTO.getAnswerInfo().setContentType(6);
}
}
}
}
logSessionInfo(robotTalkDTO, matchQuestionAnswerDTO);
// 清除中间数据
robotTalkDTO.getAnswerInfo().setVoiceBase64(null);
robotTalkDTO.getAnswerInfo().setAnswerByte(null);
robotTalkDTO.getAnswerInfo().setAnswerFile(null);
log.info("textTalk2Robot:结束问答,耗时:{}ms",timeInterval.interval("all"));
return robotTalkDTO;
}
/**
* 便
* @param input
* @return
*/
private String numberToChinese(String input){
if (StrUtil.isEmpty(input)){
return input;
}
String regex = "(\\d+(\\.\\d+)?)(?=元|万元|亿元|个)";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(input);
while (matcher.find()) {
String group = matcher.group();
if (StrUtil.isNotEmpty(group) && NumberUtil.isNumber(group)){
input = input.replace(group,Convert.numberToChinese(NumberUtil.parseDouble(group),false));
}
}
return input;
}
private void setSuspectInfoAndNextFlag(RobotTalkReq robotTalkReq, RobotTalkDTO robotTalkDTO, MatchQuestionAnswerDTO matchQuestionAnswerDTO, IrRobotConfig config) {
String sessionId = robotTalkReq.getSessionId();
if (Objects.nonNull(matchQuestionAnswerDTO)){
List<ParamCheckDTO> paramCheckDTOS = queryTemplateProcessor.paramCheck(matchQuestionAnswerDTO.getMatchQuestionCode(), sessionId,
suspectInfo2IrSessionParam(sessionId, null));
// 匹配到意图,但是参数校验不通过,尝试提取消息中的信息
if (CollUtil.isNotEmpty(paramCheckDTOS) && !Boolean.TRUE.equals(robotTalkReq.isConfirmFlag())){
String paramsJoin = paramCheckDTOS.stream().map(ParamCheckDTO::getParamName).collect(Collectors.joining(","));
log.info("匹配到的问题:{} ,需要参数:{} ,尝试抽取问题:{} 中的参数。",matchQuestionAnswerDTO.getMatchQuestionCode(),paramsJoin, robotTalkReq.getMessage());
extractInformation(sessionId, robotTalkReq.getMessage(), robotTalkDTO);
SuspectInfo suspectInfo = robotTalkDTO.getAnswerInfo().getSuspectInfo();
if (Objects.isNull(suspectInfo) || suspectInfo.isEmpty()){
// 未抽取到信息,继续下一步操作
robotTalkDTO.setDoNext(true);
}else {
robotTalkDTO.getAnswerInfo().setContentType(6);
}
}
}else {
if (!Boolean.TRUE.equals(robotTalkReq.isConfirmFlag())){
log.info("问题:{},未匹配到意图且confirmFlag标识不为true,抽取问题中的关键属性", robotTalkReq.getMessage());
// 未匹配到意图,且不是关键信息确认操作则提取信息中的关键信息
extractInformation(sessionId, robotTalkReq.getMessage(), robotTalkDTO);
}else {
// 未匹配到意图,是关键信息确认操作,不做问题响应,只做默认回答
log.info("问题:{},未匹配到意图且confirmFlag标识为true,设置默认回答:{}", robotTalkReq.getMessage(), config.getUnrecognizedTwo());
robotTalkDTO.getAnswerInfo().setMessage(config.getUnrecognizedTwo());
robotTalkDTO.setDoNext(false);
}
}
SuspectInfo suspectInfo = robotTalkDTO.getAnswerInfo().getSuspectInfo();
if (Objects.isNull(suspectInfo) || suspectInfo.isEmpty()){
robotTalkDTO.getAnswerInfo().setSuspectInfo(robotTalkReq.getTitleContent());
}
}
private void logSessionInfo(RobotTalkDTO robotTalkDTO, MatchQuestionAnswerDTO matchQuestionAnswerDTO) {
// 组装日志信息
IrSessionHistory irSessionHistory = talk2SessionHistory(robotTalkDTO.getSessionId(),
robotTalkDTO, matchQuestionAnswerDTO);
// 保存回答音频文件
IrVoice irVoice = saveAudioIfNoAbsent(robotTalkDTO.getAnswerInfo().getMessage());
if (Objects.nonNull(irVoice)) {
robotTalkDTO.getAnswerInfo().setVoiceBaseId(irVoice.getId());
robotTalkDTO.getAnswerInfo().setVoiceBase64(irVoice.getVoiceBase64());
robotTalkDTO.getAnswerInfo().setAudioLength(irVoice.getLength());
irSessionHistory.setAnswerVoiceId(irVoice.getId());
}
if (Objects.nonNull(robotTalkDTO.getAnswerInfo().getAnswerByte())
|| Objects.nonNull(robotTalkDTO.getAnswerInfo().getAnswerFile())) {
// 如果返回的结果是字节,保存到文件表
IrFile irFile = saveFileIfNoAbsent(robotTalkDTO.getAnswerInfo());
if (Objects.nonNull(irFile)) {
robotTalkDTO.getAnswerInfo().setContentType(3);
robotTalkDTO.getAnswerInfo().setByteId(irFile.getId());
irSessionHistory.setAnswerFileId(irFile.getId());
}
}
// 写入对话日志
sessionService.saveOrUpdate(irSessionHistory);
robotTalkDTO.getAskInfo().setAskId(irSessionHistory.getId());
}
private IrVoice saveAudioIfNoAbsent(String message) {
if (StrUtil.isEmpty(message)) {
return null;
}
String translate = numberToChinese(message);
TtsResultDTO resultDTO = voiceService.textToVoice(translate);
IrVoice irVoice = new IrVoice();
irVoice.setVoiceBase64(resultDTO.getAudio());
if (NumberUtil.isNumber(resultDTO.getDuration())){
irVoice.setLength(NumberUtil.parseInt(resultDTO.getDuration()));
}
irVoiceService.save(irVoice);
return irVoice;
}
private QueryProcessDTO queryMatch2Answer(String sessionId, MatchQuestionAnswerDTO matchQuestionAnswerDTO,
RobotTalkDTO robotTalkDTO, SuspectInfo suspectInfo) {
QueryProcessDTO process = QueryProcessDTO.builder().state(2).build();
if (Objects.nonNull(matchQuestionAnswerDTO) && robotTalkDTO.isDoNext()) {
process = queryTemplateProcessor.process(matchQuestionAnswerDTO.getMatchQuestionCode(), sessionId,
()->this.suspectInfo2IrSessionParam(sessionId,suspectInfo));
}
return process;
}
private List<IrSessionParam> suspectInfo2IrSessionParam(String sessionId,SuspectInfo suspectInfo) {
List<IrSessionParam> sessionParams = new ArrayList<>();
if (Objects.nonNull(suspectInfo)){
sessionParams.add(new IrSessionParam(sessionId,"jykh",StrUtil.join(",",suspectInfo.getCardNumber())));
sessionParams.add(new IrSessionParam(sessionId,"khrzjhm",suspectInfo.getIdNumber()));
sessionParams.add(new IrSessionParam(sessionId,"khrmc",suspectInfo.getName()));
}
IrSessionParam sessionParam = irSessionParamService.lambdaQuery().eq(IrSessionParam::getSessionId, sessionId)
.eq(IrSessionParam::getParamName, "ajid").one();
if (Objects.nonNull(sessionParam)) {
sessionParams.add(sessionParam);
}
return sessionParams;
}
private MatchQuestionAnswerDTO matchQuestionAnswer(String message) {
if (StrUtil.isNotEmpty(message)){
// 如果消息为空,则没必要进行意图匹配
try {
List<MatchQuestionAnswerDTO> matchQuestionAnswerDTOS = matchToolService.execMatch(message);
log.info("问题:{} 匹配到的意图:{}", message, JSONUtil.toJsonStr(matchQuestionAnswerDTOS));
return CollUtil.getFirst(matchQuestionAnswerDTOS);
} catch (Exception e) {
log.error("textTalk2Robot:内容:{} 相似度匹配失败", message,e);
return null;
}
}
return null;
}
private void extractInformation(String sessionId, String message, RobotTalkDTO robotTalkDTO) {
ExtractInformationDTO extractInformationDTO = matchToolService.extractInformation(message);
if (Objects.isNull(extractInformationDTO) || extractInformationDTO.allEmpty()) {
return;
}
// 识别到重要信息
IrSessionParam sessionParam = irSessionParamService.lambdaQuery().eq(IrSessionParam::getSessionId, sessionId).eq(IrSessionParam::getParamName, "ajid").one();
AnswerInfo answerInfo = robotTalkDTO.getAnswerInfo();
if (Objects.isNull(sessionParam) || StrUtil.isEmpty(sessionParam.getParamValue())) {
// 案件id为空需要设置案件id
answerInfo.setContentType(2);
answerInfo.setMessage("请先设置案件信息");
robotTalkDTO.setDoNext(false);
} else {
Integer caseNo = StrUtil.isEmpty(sessionParam.getParamValue()) ? null : Integer.valueOf(sessionParam.getParamValue());
List<SuspectInfo> suspectInfos = robotDataMapper.querySuspect(caseNo,
extractInformationDTO.getIdNumber(),
extractInformationDTO.getName(),
extractInformationDTO.getCardNumber());
answerInfo.setContentType(6);
SuspectInfo suspectInfo = Objects.isNull(CollUtil.getFirst(suspectInfos)) ?
extractInformation2Suspect(extractInformationDTO) : CollUtil.getFirst(suspectInfos);
answerInfo.setSuspectInfo(suspectInfo);
robotTalkDTO.setDoNext(false);
}
}
private SuspectInfo extractInformation2Suspect(ExtractInformationDTO informationDTO) {
SuspectInfo suspectInfo = new SuspectInfo();
suspectInfo.setName(informationDTO.getName());
suspectInfo.setIdNumber(informationDTO.getIdNumber());
if (StrUtil.isNotEmpty(informationDTO.getCardNumber())){
suspectInfo.setCardNumber(CollUtil.newArrayList(informationDTO.getCardNumber()));
}
return suspectInfo;
}
private static String decideAnswer(QueryProcessDTO process, IrRobotConfig config) {
// 正确查询出结果
if (Integer.valueOf(0).equals(process.getState())){
if (Integer.valueOf(1).equals(process.getContentType()) ||
Integer.valueOf(6).equals(process.getContentType())){
// 查询结果类型为字符串
return process.getStringContent();
}
}
// 参数异常或查询结果异常
if (Integer.valueOf(1).equals(process.getState())){
return StrUtil.join(",",process.paramCheckToMessage());
}
if (Integer.valueOf(2).equals(process.getState())){
return config.getUnrecognizedOne();
}
return null;
}
private IrFile saveFileIfNoAbsent(AnswerInfo answerInfo) {
if (Objects.isNull(answerInfo.getAnswerByte()) && Objects.isNull(answerInfo.getAnswerFile())){
return null;
}
String fileType = Integer.valueOf(3).equals(answerInfo.getContentType()) ? "1" : "2";
IrFile irFile = new IrFile();
irFile.setFileType(fileType);
irFile.setFileName("answer");
if ("1".equals(irFile.getFileType())){
irFile.setFileByte(answerInfo.getAnswerByte());
}else {
irFile.setFileByte(IoUtil.readBytes(FileUtil.getInputStream(answerInfo.getAnswerFile())));
FileUtil.del(answerInfo.getAnswerFile());
}
irFile.setFileSize(irFile.getFileByte().length);
irFile.setFileType(fileType);
irFileService.save(irFile);
return irFile;
}
private IrSessionHistory talk2SessionHistory(String sessionId, RobotTalkDTO robotTalkDTO,
MatchQuestionAnswerDTO matchQuestionAnswerDTO) {
IrSessionHistory sessionHistory = new IrSessionHistory();
sessionHistory.setId(robotTalkDTO.getAskInfo().getAskId());
sessionHistory.setSessionId(sessionId);
AnswerInfo answerInfo = robotTalkDTO.getAnswerInfo();
sessionHistory.setAnswer(answerInfo.getMessage());
sessionHistory.setAnswerType(answerInfo.getContentType());
sessionHistory.setUserQuestion(robotTalkDTO.getAskInfo().getMessage());
sessionHistory.setCreateUserId(UserUtil.getUser().getId());
if (Objects.nonNull(matchQuestionAnswerDTO)){
sessionHistory.setMatchKnowledgeId(matchQuestionAnswerDTO.getMatchQuestionCode());
sessionHistory.setThreshold(BigDecimal.valueOf(matchQuestionAnswerDTO.getMatchScore()));
}
return sessionHistory;
}
@Override
@Transactional(rollbackFor = Exception.class)
public RobotTalkDTO videoTalk2Robot(MultipartFile multipartFile,RobotTalkReq robotTalkReq) {
String sessionId = robotTalkReq.getSessionId();
Assert.notEmpty(sessionId, "sessionId不能为空");
Assert.notNull(multipartFile, "multipartFile不能为空");
byte[] bytes = null;
try {
bytes = multipartFile.getBytes();
String message = voiceService.voiceToText(bytes, sessionId);
robotTalkReq.setMessage(message);
} catch (IOException e) {
log.error("语音转文字失败", e);
}
RobotTalkDTO robotTalkDTO = this.textTalk2Robot(robotTalkReq);
robotTalkDTO.getAskInfo().setContentType(2);
if (Objects.nonNull(bytes)){
IrVoice irVoice = new IrVoice();
irVoice.setVoiceBase64(Base64.encode(bytes));
irVoiceService.save(irVoice);
if (StrUtil.isNotEmpty(robotTalkDTO.getAskInfo().getAskId())){
sessionService.lambdaUpdate()
.eq(IrSessionHistory::getId,robotTalkDTO.getAskInfo().getAskId())
.set(IrSessionHistory::getUserQuestionVoiceId,irVoice.getId()).update();
}
}
return robotTalkDTO;
}
@Override
public List<CommonDialogVo> talkList(String sessionId) {
Assert.notEmpty(sessionId, "sessionId不能为空");
List<IrSessionHistory> sessionHistoryList = sessionService.lambdaQuery().eq(IrSessionHistory::getSessionId, sessionId).orderBy(true, true,IrSessionHistory::getCreateTime).list();
// 获取音频的长度
List<String> voiceIds = sessionHistoryList.stream()
.map(session -> CollUtil.toList(session.getAnswerVoiceId(), session.getUserQuestionVoiceId()))
.flatMap(Collection::stream).filter(StrUtil::isNotEmpty).collect(Collectors.toList());
Map<String, Integer> voiceLengthMap = CollUtil.isEmpty(voiceIds) ? new HashMap<>() :
irVoiceService.listByIds(voiceIds).stream().filter(i->Objects.nonNull(i.getLength())).collect(Collectors.toMap(IrVoice::getId, IrVoice::getLength));
return sessionHistoryList.stream().map(session->this.sessionHistory2RobotTalkDTO(session,voiceLengthMap))
.filter(Objects::nonNull).flatMap(Collection::stream).collect(Collectors.toList());
}
private List<CommonDialogVo> sessionHistory2RobotTalkDTO(IrSessionHistory sessionHistory,Map<String, Integer> voiceLengthMap) {
if (Objects.isNull(sessionHistory)){
return null;
}
CommonDialogVo askDiaLog = CommonDialogVo.builder().dialogType(1)
.dialogId(sessionHistory.getId()).sessionId(sessionHistory.getSessionId())
.contentType(StrUtil.isEmpty(sessionHistory.getUserQuestionVoiceId()) ? 1 : 2)//userQuestionVoiceId 为空则为1 否则为2
.message(sessionHistory.getUserQuestion()).voiceBaseId(sessionHistory.getUserQuestionVoiceId())
.audioLength(voiceLengthMap.get(sessionHistory.getUserQuestionVoiceId())).build();
if (Integer.valueOf(6).equals(sessionHistory.getAnswerType())){
// 屏蔽弹框记录
return CollUtil.newArrayList(askDiaLog);
}
CommonDialogVo answerDiaLog = CommonDialogVo.builder().dialogType(2)
.dialogId(sessionHistory.getId()).sessionId(sessionHistory.getSessionId())
.contentType(sessionHistory.getAnswerType()).message(sessionHistory.getAnswer())
.byteId(sessionHistory.getAnswerFileId()).voiceBaseId(sessionHistory.getAnswerVoiceId())
.audioLength(voiceLengthMap.get(sessionHistory.getAnswerVoiceId()))
.build();
return CollUtil.toList(askDiaLog,answerDiaLog);
}
@Override
public void getAudio(HttpServletResponse response, String audioId) throws IOException {
Assert.notEmpty(audioId, "audioId不能为空");
IrVoice voice = irVoiceService.getById(audioId);
if (Objects.isNull(voice) || StrUtil.isEmpty(voice.getVoiceBase64())){
return;
}
Base64.decodeToStream(voice.getVoiceBase64(), response.getOutputStream(),false);
}
@Override
public void downLoadArchives(HttpServletResponse response,ArchivesReqVo archivesReq) throws IOException {
String path = localArchivesFilePath(archivesReq);
if (StrUtil.isEmpty(path)){
return;
}
IoUtil.copy(Files.newInputStream(Paths.get(path)),response.getOutputStream());
}
@Override
public void downLoadFile(HttpServletResponse response, String fileType, String imageId) throws IOException {
IrFile file = irFileService.getById(imageId);
if (Objects.isNull(file) || Objects.isNull(file.getFileByte())){
return;
}
if ("1".equals(file.getFileType())){
IoUtil.write(response.getOutputStream(), true, Base64.decode(file.getFileByte()));
}
if ("2".equals(file.getFileType())){
IoUtil.write(response.getOutputStream(), true,file.getFileByte());
}
}
@Override
public List<SuspectInfo> querySuspectInfo(String sessionId, String name) {
Assert.notEmpty(sessionId, "sessionId不能为空");
//Assert.notEmpty(name, "name不能为空");
IrSessionParam sessionParam = irSessionParamService.lambdaQuery().eq(IrSessionParam::getSessionId, sessionId).eq(IrSessionParam::getParamName, "ajid").one();
Assert.notNull(sessionParam, "未查询到案件");
Assert.notEmpty(sessionParam.getParamValue(), "ajid不能为空");
return robotDataMapper.querySuspect(NumberUtil.parseInt(sessionParam.getParamValue()), null, name, null);
}
private String localArchivesFilePath(ArchivesReqVo archivesReq) {
String path = StrUtil.join(File.separator,archivesReq.getCaseId(),
StrUtil.join("-",archivesReq.getName(),archivesReq.getIdNumber()),
archivesReq.getDocumentId());
// 判断文件是否存在
if (FileUtil.exist(path)){
return path;
}
return null;
}
}