package com.supervision.service.impl; import cn.hutool.core.codec.Base64; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.convert.Convert; import cn.hutool.core.date.TimeInterval; import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; import cn.hutool.core.lang.Assert; import cn.hutool.core.util.NumberUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import com.supervision.domain.*; import com.supervision.dto.paddlespeech.res.TtsResultDTO; import com.supervision.dto.robot.*; import com.supervision.dto.matchTool.ExtractInformationDTO; import com.supervision.dto.matchTool.MatchQuestionAnswerDTO; import com.supervision.dto.QueryProcessDTO; import com.supervision.mapper.RobotDataMapper; import com.supervision.service.*; import com.supervision.util.UserUtil; import com.supervision.vo.robot.ArchivesReqVo; import com.supervision.vo.robot.CommonDialogVo; import com.supervision.vo.talk.RobotTalkReq; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Files; import java.nio.file.Paths; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @Slf4j @Service @RequiredArgsConstructor public class RobotTalkServiceImpl implements RobotTalkService { private final QueryTemplateProcessor queryTemplateProcessor; private final MatchToolService matchToolService; private final VoiceService voiceService; private final IrRobotConfigService irRobotConfigService; private final IrSessionHistoryService sessionService; private final IrSessionParamService irSessionParamService; private final IrVoiceService irVoiceService; private final IrFileService irFileService; @Resource private final RobotDataMapper robotDataMapper; @Override @Transactional(rollbackFor = Exception.class) public RobotTalkDTO textTalk2Robot(RobotTalkReq robotTalkReq) { String sessionId = robotTalkReq.getSessionId(); Assert.notEmpty(sessionId, "sessionId不能为空"); TimeInterval timeInterval = new TimeInterval(); timeInterval.start("all"); log.info("textTalk2Robot:开始问答,参数:{}",JSONUtil.toJsonStr(robotTalkReq)); RobotTalkDTO robotTalkDTO = RobotTalkDTO.builder() .sessionId(sessionId).doNext(true) .askInfo(AskInfo.builder().contentType(1).message(robotTalkReq.getMessage()).build()) .answerInfo(AnswerInfo.builder().contentType(robotTalkReq.getAnswerType()).build()) .build(); // 获取机器人配置 IrRobotConfig config = irRobotConfigService.lambdaQuery().one(); // 确认弹框信息 if (StrUtil.isNotEmpty(robotTalkReq.getAskId())){ IrSessionHistory sessionHistory = sessionService.lambdaQuery().eq(IrSessionHistory::getId, robotTalkReq.getAskId()).one(); if (Objects.nonNull(sessionHistory)){ robotTalkReq.setMessage(sessionHistory.getUserQuestion()); robotTalkReq.setConfirmFlag(true); robotTalkDTO.getAskInfo().setAskId(robotTalkReq.getAskId()); } } // 匹配问题意图 MatchQuestionAnswerDTO matchQuestionAnswerDTO = matchQuestionAnswer(robotTalkReq.getMessage()); setSuspectInfoAndNextFlag(robotTalkReq, robotTalkDTO, matchQuestionAnswerDTO,config); if (Objects.isNull(matchQuestionAnswerDTO) && robotTalkDTO.isDoNext()){ // 未匹配到查询意图,设置默认错误语 log.info("问题:{}未匹配到意图",robotTalkReq.getMessage()); robotTalkDTO.getAnswerInfo().setMessage(config.getErrorLanguage()); robotTalkDTO.setDoNext(false); } if (robotTalkDTO.isDoNext()){ QueryProcessDTO matchAnswer = queryMatch2Answer(sessionId, matchQuestionAnswerDTO, robotTalkDTO, robotTalkReq.getTitleContent()); if (Integer.valueOf(0).equals(matchAnswer.getState()) && Integer.valueOf(3).equals(matchAnswer.getContentType()) && matchAnswer.getByteContent() != null){ // 查询结果类型为字节数组 robotTalkDTO.getAnswerInfo().setContentType(3); robotTalkDTO.getAnswerInfo().setAnswerByte(matchAnswer.getByteContent()); }else{ robotTalkDTO.getAnswerInfo().setAnswerFile(matchAnswer.getFileContent()); robotTalkDTO.getAnswerInfo().setMessage(decideAnswer(matchAnswer, config)); if (matchAnswer.getState() == 1){ if (!Boolean.TRUE.equals(robotTalkReq.isConfirmFlag())){ robotTalkDTO.getAnswerInfo().setContentType(6); } } } } logSessionInfo(robotTalkDTO, matchQuestionAnswerDTO); // 清除中间数据 robotTalkDTO.getAnswerInfo().setVoiceBase64(null); robotTalkDTO.getAnswerInfo().setAnswerByte(null); robotTalkDTO.getAnswerInfo().setAnswerFile(null); log.info("textTalk2Robot:结束问答,耗时:{}ms",timeInterval.interval("all")); return robotTalkDTO; } /** * 转换语句中的数字,方便语音合成发音 * @param input * @return */ private String numberToChinese(String input){ if (StrUtil.isEmpty(input)){ return input; } String regex = "(\\d+(\\.\\d+)?)(?=元|万元|亿元|个)"; Pattern pattern = Pattern.compile(regex); Matcher matcher = pattern.matcher(input); while (matcher.find()) { String group = matcher.group(); if (StrUtil.isNotEmpty(group) && NumberUtil.isNumber(group)){ input = input.replace(group,Convert.numberToChinese(NumberUtil.parseDouble(group),false)); } } return input; } private void setSuspectInfoAndNextFlag(RobotTalkReq robotTalkReq, RobotTalkDTO robotTalkDTO, MatchQuestionAnswerDTO matchQuestionAnswerDTO, IrRobotConfig config) { String sessionId = robotTalkReq.getSessionId(); if (Objects.nonNull(matchQuestionAnswerDTO)){ List paramCheckDTOS = queryTemplateProcessor.paramCheck(matchQuestionAnswerDTO.getMatchQuestionCode(), sessionId, suspectInfo2IrSessionParam(sessionId, null)); // 匹配到意图,但是参数校验不通过,尝试提取消息中的信息 if (CollUtil.isNotEmpty(paramCheckDTOS) && !Boolean.TRUE.equals(robotTalkReq.isConfirmFlag())){ String paramsJoin = paramCheckDTOS.stream().map(ParamCheckDTO::getParamName).collect(Collectors.joining(",")); log.info("匹配到的问题:{} ,需要参数:{} ,尝试抽取问题:{} 中的参数。",matchQuestionAnswerDTO.getMatchQuestionCode(),paramsJoin, robotTalkReq.getMessage()); extractInformation(sessionId, robotTalkReq.getMessage(), robotTalkDTO); SuspectInfo suspectInfo = robotTalkDTO.getAnswerInfo().getSuspectInfo(); if (Objects.isNull(suspectInfo) || suspectInfo.isEmpty()){ // 未抽取到信息,继续下一步操作 robotTalkDTO.setDoNext(true); }else { robotTalkDTO.getAnswerInfo().setContentType(6); } } }else { if (!Boolean.TRUE.equals(robotTalkReq.isConfirmFlag())){ log.info("问题:{},未匹配到意图,且confirmFlag标识不为true,抽取问题中的关键属性", robotTalkReq.getMessage()); // 未匹配到意图,且不是关键信息确认操作则提取信息中的关键信息 extractInformation(sessionId, robotTalkReq.getMessage(), robotTalkDTO); }else { // 未匹配到意图,是关键信息确认操作,不做问题响应,只做默认回答 log.info("问题:{},未匹配到意图,且confirmFlag标识为true,设置默认回答:{}", robotTalkReq.getMessage(), config.getUnrecognizedTwo()); robotTalkDTO.getAnswerInfo().setMessage(config.getUnrecognizedTwo()); robotTalkDTO.setDoNext(false); } } SuspectInfo suspectInfo = robotTalkDTO.getAnswerInfo().getSuspectInfo(); if (Objects.isNull(suspectInfo) || suspectInfo.isEmpty()){ robotTalkDTO.getAnswerInfo().setSuspectInfo(robotTalkReq.getTitleContent()); } } private void logSessionInfo(RobotTalkDTO robotTalkDTO, MatchQuestionAnswerDTO matchQuestionAnswerDTO) { // 组装日志信息 IrSessionHistory irSessionHistory = talk2SessionHistory(robotTalkDTO.getSessionId(), robotTalkDTO, matchQuestionAnswerDTO); // 保存回答音频文件 IrVoice irVoice = saveAudioIfNoAbsent(robotTalkDTO.getAnswerInfo().getMessage()); if (Objects.nonNull(irVoice)) { robotTalkDTO.getAnswerInfo().setVoiceBaseId(irVoice.getId()); robotTalkDTO.getAnswerInfo().setVoiceBase64(irVoice.getVoiceBase64()); robotTalkDTO.getAnswerInfo().setAudioLength(irVoice.getLength()); irSessionHistory.setAnswerVoiceId(irVoice.getId()); } if (Objects.nonNull(robotTalkDTO.getAnswerInfo().getAnswerByte()) || Objects.nonNull(robotTalkDTO.getAnswerInfo().getAnswerFile())) { // 如果返回的结果是字节,保存到文件表 IrFile irFile = saveFileIfNoAbsent(robotTalkDTO.getAnswerInfo()); if (Objects.nonNull(irFile)) { robotTalkDTO.getAnswerInfo().setContentType(3); robotTalkDTO.getAnswerInfo().setByteId(irFile.getId()); irSessionHistory.setAnswerFileId(irFile.getId()); } } // 写入对话日志 sessionService.saveOrUpdate(irSessionHistory); robotTalkDTO.getAskInfo().setAskId(irSessionHistory.getId()); } private IrVoice saveAudioIfNoAbsent(String message) { if (StrUtil.isEmpty(message)) { return null; } String translate = numberToChinese(message); TtsResultDTO resultDTO = voiceService.textToVoice(translate); IrVoice irVoice = new IrVoice(); irVoice.setVoiceBase64(resultDTO.getAudio()); if (NumberUtil.isNumber(resultDTO.getDuration())){ irVoice.setLength(NumberUtil.parseInt(resultDTO.getDuration())); } irVoiceService.save(irVoice); return irVoice; } private QueryProcessDTO queryMatch2Answer(String sessionId, MatchQuestionAnswerDTO matchQuestionAnswerDTO, RobotTalkDTO robotTalkDTO, SuspectInfo suspectInfo) { QueryProcessDTO process = QueryProcessDTO.builder().state(2).build(); if (Objects.nonNull(matchQuestionAnswerDTO) && robotTalkDTO.isDoNext()) { process = queryTemplateProcessor.process(matchQuestionAnswerDTO.getMatchQuestionCode(), sessionId, ()->this.suspectInfo2IrSessionParam(sessionId,suspectInfo)); } return process; } private List suspectInfo2IrSessionParam(String sessionId,SuspectInfo suspectInfo) { List sessionParams = new ArrayList<>(); if (Objects.nonNull(suspectInfo)){ sessionParams.add(new IrSessionParam(sessionId,"jykh",StrUtil.join(",",suspectInfo.getCardNumber()))); sessionParams.add(new IrSessionParam(sessionId,"khrzjhm",suspectInfo.getIdNumber())); sessionParams.add(new IrSessionParam(sessionId,"khrmc",suspectInfo.getName())); } IrSessionParam sessionParam = irSessionParamService.lambdaQuery().eq(IrSessionParam::getSessionId, sessionId) .eq(IrSessionParam::getParamName, "ajid").one(); if (Objects.nonNull(sessionParam)) { sessionParams.add(sessionParam); } return sessionParams; } private MatchQuestionAnswerDTO matchQuestionAnswer(String message) { if (StrUtil.isNotEmpty(message)){ // 如果消息为空,则没必要进行意图匹配 try { List matchQuestionAnswerDTOS = matchToolService.execMatch(message); log.info("问题:{} 匹配到的意图:{}", message, JSONUtil.toJsonStr(matchQuestionAnswerDTOS)); return CollUtil.getFirst(matchQuestionAnswerDTOS); } catch (Exception e) { log.error("textTalk2Robot:内容:{} 相似度匹配失败", message,e); return null; } } return null; } private void extractInformation(String sessionId, String message, RobotTalkDTO robotTalkDTO) { ExtractInformationDTO extractInformationDTO = matchToolService.extractInformation(message); if (Objects.isNull(extractInformationDTO) || extractInformationDTO.allEmpty()) { return; } // 识别到重要信息 IrSessionParam sessionParam = irSessionParamService.lambdaQuery().eq(IrSessionParam::getSessionId, sessionId).eq(IrSessionParam::getParamName, "ajid").one(); AnswerInfo answerInfo = robotTalkDTO.getAnswerInfo(); if (Objects.isNull(sessionParam) || StrUtil.isEmpty(sessionParam.getParamValue())) { // 案件id为空,需要设置案件id answerInfo.setContentType(2); answerInfo.setMessage("请先设置案件信息"); robotTalkDTO.setDoNext(false); } else { Integer caseNo = StrUtil.isEmpty(sessionParam.getParamValue()) ? null : Integer.valueOf(sessionParam.getParamValue()); List suspectInfos = robotDataMapper.querySuspect(caseNo, extractInformationDTO.getIdNumber(), extractInformationDTO.getName(), extractInformationDTO.getCardNumber()); answerInfo.setContentType(6); SuspectInfo suspectInfo = Objects.isNull(CollUtil.getFirst(suspectInfos)) ? extractInformation2Suspect(extractInformationDTO) : CollUtil.getFirst(suspectInfos); answerInfo.setSuspectInfo(suspectInfo); robotTalkDTO.setDoNext(false); } } private SuspectInfo extractInformation2Suspect(ExtractInformationDTO informationDTO) { SuspectInfo suspectInfo = new SuspectInfo(); suspectInfo.setName(informationDTO.getName()); suspectInfo.setIdNumber(informationDTO.getIdNumber()); if (StrUtil.isNotEmpty(informationDTO.getCardNumber())){ suspectInfo.setCardNumber(CollUtil.newArrayList(informationDTO.getCardNumber())); } return suspectInfo; } private static String decideAnswer(QueryProcessDTO process, IrRobotConfig config) { // 正确查询出结果 if (Integer.valueOf(0).equals(process.getState())){ if (Integer.valueOf(1).equals(process.getContentType()) || Integer.valueOf(6).equals(process.getContentType())){ // 查询结果类型为字符串 return process.getStringContent(); } } // 参数异常或查询结果异常 if (Integer.valueOf(1).equals(process.getState())){ return StrUtil.join(",",process.paramCheckToMessage()); } if (Integer.valueOf(2).equals(process.getState())){ return config.getUnrecognizedOne(); } return null; } private IrFile saveFileIfNoAbsent(AnswerInfo answerInfo) { if (Objects.isNull(answerInfo.getAnswerByte()) && Objects.isNull(answerInfo.getAnswerFile())){ return null; } String fileType = Integer.valueOf(3).equals(answerInfo.getContentType()) ? "1" : "2"; IrFile irFile = new IrFile(); irFile.setFileType(fileType); irFile.setFileName("answer"); if ("1".equals(irFile.getFileType())){ irFile.setFileByte(answerInfo.getAnswerByte()); }else { irFile.setFileByte(IoUtil.readBytes(FileUtil.getInputStream(answerInfo.getAnswerFile()))); FileUtil.del(answerInfo.getAnswerFile()); } irFile.setFileSize(irFile.getFileByte().length); irFile.setFileType(fileType); irFileService.save(irFile); return irFile; } private IrSessionHistory talk2SessionHistory(String sessionId, RobotTalkDTO robotTalkDTO, MatchQuestionAnswerDTO matchQuestionAnswerDTO) { IrSessionHistory sessionHistory = new IrSessionHistory(); sessionHistory.setId(robotTalkDTO.getAskInfo().getAskId()); sessionHistory.setSessionId(sessionId); AnswerInfo answerInfo = robotTalkDTO.getAnswerInfo(); sessionHistory.setAnswer(answerInfo.getMessage()); sessionHistory.setAnswerType(answerInfo.getContentType()); sessionHistory.setUserQuestion(robotTalkDTO.getAskInfo().getMessage()); sessionHistory.setCreateUserId(UserUtil.getUser().getId()); if (Objects.nonNull(matchQuestionAnswerDTO)){ sessionHistory.setMatchKnowledgeId(matchQuestionAnswerDTO.getMatchQuestionCode()); sessionHistory.setThreshold(BigDecimal.valueOf(matchQuestionAnswerDTO.getMatchScore())); } return sessionHistory; } @Override @Transactional(rollbackFor = Exception.class) public RobotTalkDTO videoTalk2Robot(MultipartFile multipartFile,RobotTalkReq robotTalkReq) { String sessionId = robotTalkReq.getSessionId(); Assert.notEmpty(sessionId, "sessionId不能为空"); Assert.notNull(multipartFile, "multipartFile不能为空"); byte[] bytes = null; try { bytes = multipartFile.getBytes(); String message = voiceService.voiceToText(bytes, sessionId); robotTalkReq.setMessage(message); } catch (IOException e) { log.error("语音转文字失败", e); } RobotTalkDTO robotTalkDTO = this.textTalk2Robot(robotTalkReq); robotTalkDTO.getAskInfo().setContentType(2); if (Objects.nonNull(bytes)){ IrVoice irVoice = new IrVoice(); irVoice.setVoiceBase64(Base64.encode(bytes)); irVoiceService.save(irVoice); if (StrUtil.isNotEmpty(robotTalkDTO.getAskInfo().getAskId())){ sessionService.lambdaUpdate() .eq(IrSessionHistory::getId,robotTalkDTO.getAskInfo().getAskId()) .set(IrSessionHistory::getUserQuestionVoiceId,irVoice.getId()).update(); } } return robotTalkDTO; } @Override public List talkList(String sessionId) { Assert.notEmpty(sessionId, "sessionId不能为空"); List sessionHistoryList = sessionService.lambdaQuery().eq(IrSessionHistory::getSessionId, sessionId).orderBy(true, true,IrSessionHistory::getCreateTime).list(); // 获取音频的长度 List voiceIds = sessionHistoryList.stream() .map(session -> CollUtil.toList(session.getAnswerVoiceId(), session.getUserQuestionVoiceId())) .flatMap(Collection::stream).filter(StrUtil::isNotEmpty).collect(Collectors.toList()); Map voiceLengthMap = CollUtil.isEmpty(voiceIds) ? new HashMap<>() : irVoiceService.listByIds(voiceIds).stream().filter(i->Objects.nonNull(i.getLength())).collect(Collectors.toMap(IrVoice::getId, IrVoice::getLength)); return sessionHistoryList.stream().map(session->this.sessionHistory2RobotTalkDTO(session,voiceLengthMap)) .filter(Objects::nonNull).flatMap(Collection::stream).collect(Collectors.toList()); } private List sessionHistory2RobotTalkDTO(IrSessionHistory sessionHistory,Map voiceLengthMap) { if (Objects.isNull(sessionHistory)){ return null; } CommonDialogVo askDiaLog = CommonDialogVo.builder().dialogType(1) .dialogId(sessionHistory.getId()).sessionId(sessionHistory.getSessionId()) .contentType(StrUtil.isEmpty(sessionHistory.getUserQuestionVoiceId()) ? 1 : 2)//userQuestionVoiceId 为空,则为1 否则为2 .message(sessionHistory.getUserQuestion()).voiceBaseId(sessionHistory.getUserQuestionVoiceId()) .audioLength(voiceLengthMap.get(sessionHistory.getUserQuestionVoiceId())).build(); if (Integer.valueOf(6).equals(sessionHistory.getAnswerType())){ // 屏蔽弹框记录 return CollUtil.newArrayList(askDiaLog); } CommonDialogVo answerDiaLog = CommonDialogVo.builder().dialogType(2) .dialogId(sessionHistory.getId()).sessionId(sessionHistory.getSessionId()) .contentType(sessionHistory.getAnswerType()).message(sessionHistory.getAnswer()) .byteId(sessionHistory.getAnswerFileId()).voiceBaseId(sessionHistory.getAnswerVoiceId()) .audioLength(voiceLengthMap.get(sessionHistory.getAnswerVoiceId())) .build(); return CollUtil.toList(askDiaLog,answerDiaLog); } @Override public void getAudio(HttpServletResponse response, String audioId) throws IOException { Assert.notEmpty(audioId, "audioId不能为空"); IrVoice voice = irVoiceService.getById(audioId); if (Objects.isNull(voice) || StrUtil.isEmpty(voice.getVoiceBase64())){ return; } Base64.decodeToStream(voice.getVoiceBase64(), response.getOutputStream(),false); } @Override public void downLoadArchives(HttpServletResponse response,ArchivesReqVo archivesReq) throws IOException { String path = localArchivesFilePath(archivesReq); if (StrUtil.isEmpty(path)){ return; } IoUtil.copy(Files.newInputStream(Paths.get(path)),response.getOutputStream()); } @Override public void downLoadFile(HttpServletResponse response, String fileType, String imageId) throws IOException { IrFile file = irFileService.getById(imageId); if (Objects.isNull(file) || Objects.isNull(file.getFileByte())){ return; } if ("1".equals(file.getFileType())){ IoUtil.write(response.getOutputStream(), true, Base64.decode(file.getFileByte())); } if ("2".equals(file.getFileType())){ IoUtil.write(response.getOutputStream(), true,file.getFileByte()); } } @Override public List querySuspectInfo(String sessionId, String name) { Assert.notEmpty(sessionId, "sessionId不能为空"); //Assert.notEmpty(name, "name不能为空"); IrSessionParam sessionParam = irSessionParamService.lambdaQuery().eq(IrSessionParam::getSessionId, sessionId).eq(IrSessionParam::getParamName, "ajid").one(); Assert.notNull(sessionParam, "未查询到案件"); Assert.notEmpty(sessionParam.getParamValue(), "ajid不能为空"); return robotDataMapper.querySuspect(NumberUtil.parseInt(sessionParam.getParamValue()), null, name, null); } private String localArchivesFilePath(ArchivesReqVo archivesReq) { String path = StrUtil.join(File.separator,archivesReq.getCaseId(), StrUtil.join("-",archivesReq.getName(),archivesReq.getIdNumber()), archivesReq.getDocumentId()); // 判断文件是否存在 if (FileUtil.exist(path)){ return path; } return null; } }