Merge branch 'dev-chat' into dev

jinan_dev
xueqingkun 5 months ago
commit 851c50cb13

@ -186,7 +186,7 @@ public class CaseEvidenceController {
return R.ok(directoryName);
}
@Operation(summary = "查询证据详情-结果以树的方式展示")
@Operation(summary = "迁移证据信息")
@GetMapping("/fresh")
public R<String> generateDirectoryName() {
caseEvidenceService.refreshCaseEvidence();

@ -26,6 +26,8 @@ public class ModelCase implements Serializable {
*/
@TableId
private String id;
private String knowledgeBaseId;
/**
*

@ -49,5 +49,14 @@ public interface ModelCaseService extends IService<ModelCase> {
IPage<IndexDetail> getIndexDetail(IndexResultQuery query, Integer page, Integer size);
void initCaseKnowledgeBase(String caseId);
/**
*
*/
void migrateRecordKnowledgeBase();
}

@ -13,5 +13,8 @@ public interface NoteRecordService extends IService<NoteRecord> {
void uploadFileToLangChainChat(String caseId);
void uploadRecordFileToDifyKnowledgeBase(String caseId);
String saveOrUpdRecord(NoteRecord noteRecord);
}

@ -3,6 +3,8 @@ package com.supervision.police.service.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.bean.copier.CopyOptions;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.TimeInterval;
import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.StrUtil;
@ -14,8 +16,6 @@ import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.chat.client.LangChainChatService;
import com.supervision.chat.client.dto.CreateBaseDTO;
import com.supervision.chat.client.dto.LangChainChatRes;
import com.supervision.common.domain.R;
import com.supervision.common.enums.ResultStatusEnum;
import com.supervision.common.exception.CustomException;
@ -28,6 +28,7 @@ import com.supervision.police.dto.*;
import com.supervision.police.mapper.ModelCaseMapper;
import com.supervision.police.service.*;
import com.supervision.police.vo.ModelCaseVO;
import com.supervision.utils.DifyApiUtil;
import com.supervision.utils.IndexRuleUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -76,6 +77,8 @@ public class ModelCaseServiceImpl extends ServiceImpl<ModelCaseMapper, ModelCase
private final EvidenceDirectoryService directoryService;
private final DifyApiUtil difyApiUtil;
/**
*
*
@ -166,6 +169,12 @@ public class ModelCaseServiceImpl extends ServiceImpl<ModelCaseMapper, ModelCase
Long count = super.lambdaQuery().eq(ModelCase::getCaseNo, modelCase.getCaseNo()).count();
Assert.isTrue(count == 0, "案件编号已存在,请更换案件编号!");
// 这里需要调用知识库的接口,去保存知识库
String databaseId = difyApiUtil.createDataset(modelCase.getCaseName(), modelCase.getCaseDetail());
if (StrUtil.isEmpty(databaseId)) {
throw new BusinessException("保存知识库失败");
}
modelCase.setKnowledgeBaseId(databaseId);
Long num = modelCaseMapper.selectCount(null);
modelCase.setIndexNum(Integer.parseInt(num.toString()) + 1);
i = modelCaseMapper.insert(modelCase);
@ -173,15 +182,6 @@ public class ModelCaseServiceImpl extends ServiceImpl<ModelCaseMapper, ModelCase
// 保存案件行为人
casePersonService.saveCaseActor(modelCase.getId(), modelCaseBase.getCaseActorName(), modelCaseBase.getCaseActorIdCard());
caseEvidenceService.initCaseEvidenceDirectory(modelCase.getId(), modelCase.getCaseType());
// 这里需要调用知识库的接口,去保存知识库
CreateBaseDTO createBaseDTO = new CreateBaseDTO();
createBaseDTO.setKnowledge_base_name(modelCase.getCaseNo());
LangChainChatRes chat = langChainChatService.createBase(createBaseDTO);
log.info("创建知识库:{}", chat);
if (200 != chat.getCode()) {
throw new BusinessException("保存知识库失败");
}
}
if (i > 0) {
return R.okMsg("保存成功");
@ -208,13 +208,9 @@ public class ModelCaseServiceImpl extends ServiceImpl<ModelCaseMapper, ModelCase
public R<?> realDeleteByIds(List<String> ids) {
List<ModelCase> modelCases = modelCaseMapper.selectBatchIds(ids);
if (modelCases != null && !modelCases.isEmpty()) {
modelCases.forEach(modelCase -> {
LangChainChatRes<Object> langChainChatRes = langChainChatService.deleteBase(modelCase.getCaseNo());
if (200 != langChainChatRes.getCode()) {
log.error("删除知识库失败:{}, caseNo:{}", langChainChatRes.getMsg(), modelCase.getCaseNo());
// throw new BusinessException("删除知识库失败");
}
});
modelCases.stream()
.filter(modelCase -> StrUtil.isNotEmpty(modelCase.getKnowledgeBaseId()))
.forEach(modelCase -> difyApiUtil.deleteDataset(modelCase.getKnowledgeBaseId()));
}
noteRecordService.list(new QueryWrapper<NoteRecord>().in("case_id", ids)).forEach(noteRecord -> noteRecordSplitService.delRecords(noteRecord.getId()));
List<String> personIds = casePersonService.list(new QueryWrapper<CasePerson>().in("case_id", ids)).stream().map(CasePerson::getId).collect(Collectors.toList());
@ -362,7 +358,7 @@ public class ModelCaseServiceImpl extends ServiceImpl<ModelCaseMapper, ModelCase
}
private Map<String, Object> errorMapBuilder(ModelCase modelCase, String errorText) {
return new HashMap<String, Object>(2) {{
return new HashMap<>(2) {{
put("caseNo", modelCase.getCaseNo());
put("caseName", modelCase.getCaseName());
put("errorText", errorText);
@ -384,6 +380,56 @@ public class ModelCaseServiceImpl extends ServiceImpl<ModelCaseMapper, ModelCase
return iPage;
}
@Override
public void initCaseKnowledgeBase(String caseId) {
ModelCase modelCase = this.getById(caseId);
if (StrUtil.isEmpty(modelCase.getKnowledgeBaseId())){
log.info("案件:{} 尚未创建知识库,即将创建知识库库...",modelCase.getCaseName());
String knowledgeBaseId = difyApiUtil.createDataset(
StrUtil.join("_", modelCase.getCaseName(), modelCase.getCaseNo()), modelCase.getCaseDetail());
if (StrUtil.isEmpty(knowledgeBaseId)){
log.error("案件:{} 创建支持库失败...",modelCase.getCaseName());
return;
}
modelCase.setKnowledgeBaseId(knowledgeBaseId);
modelCaseMapper.updateById(modelCase);
log.info("案件:{} 创建支持库成功知识库id为{}",modelCase.getCaseName(),modelCase.getKnowledgeBaseId());
}else {
log.info("案件:{} 已创建知识库,不进行创建...",modelCase.getCaseName());
}
log.info("案件:{} 开始上传笔录到支持库...",modelCase.getCaseName());
noteRecordService.uploadRecordFileToDifyKnowledgeBase(caseId);
log.info("案件:{} 上传笔录到支持库成功...",modelCase.getCaseName());
}
@Override
public void migrateRecordKnowledgeBase() {
// 获取所有案件
int success = 0;
int fail = 0;
// 只 同步 已经分析的案件
List<ModelCase> allModelCase = this.lambdaQuery().isNotNull(ModelCase::getCaseAnalysisSuccessTime).list();
log.info("===========>>>>>案件总数:{},开始迁移笔录到支持库...<<<<<===========",allModelCase.size());
TimeInterval timer = DateUtil.timer();
timer.start();
for (ModelCase modelCase : allModelCase) {
try {
log.info("开始迁移案件:{} 笔录到支持库,当前总体进度:{}...",modelCase.getCaseName(),NumberUtil.formatPercent((success + fail)/(allModelCase.size()*1.0), 2));
timer.start(modelCase.getId());
initCaseKnowledgeBase(modelCase.getId());
log.info("案件:{} 迁移笔录到支持库成功,耗时:{}秒...",modelCase.getCaseName(),timer.intervalSecond(modelCase.getId()));
success++;
} catch (Exception e) {
log.error("案件:{} 迁移笔录到支持库失败..",modelCase.getCaseName(),e);
fail++;
}
}
log.info("===========>>>>>迁移笔录到支持库完成,成功个数:{},失败个数:{},总耗时:{}秒...<<<<<===========",success,allModelCase.size()-success,timer.intervalSecond());
}
/**
*

@ -171,7 +171,7 @@ public class ModelServiceImpl implements ModelService {
calculateFinalScore(analyseCaseDTO, modelCase, atomicResultMap);
caseStatusManageService.whenAnalyseCaseSuccess(analyseCaseDTO.getCaseId(), modelCase.getTotalScore());
// 计算完成之后,把所有的笔录上传到模型
noteRecordService.uploadFileToLangChainChat(analyseCaseDTO.getCaseId());
noteRecordService.uploadRecordFileToDifyKnowledgeBase(analyseCaseDTO.getCaseId());
return R.ok();
}
@ -323,7 +323,7 @@ public class ModelServiceImpl implements ModelService {
modelCase.setTotalScore(max);
log.info("更新案件得分情况。最终得分:{}分(共性+入罪/共性+出罪 取最大值)。入罪:{}分。出罪:{}分。共性:{}分。", max, rz, cz, gx);
caseStatusManageService.whenAnalyseCaseSuccess(analyseCaseDTO.getCaseId(), modelCase.getTotalScore());
noteRecordService.uploadFileToLangChainChat(analyseCaseDTO.getCaseId());
noteRecordService.uploadRecordFileToDifyKnowledgeBase(analyseCaseDTO.getCaseId());
return R.ok();
}

@ -1,6 +1,7 @@
package com.supervision.police.service.impl;
import cn.hutool.json.JSONUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.chat.UploadParamEnum;
import com.supervision.chat.client.CustomMultipartFile;
@ -8,7 +9,6 @@ import com.supervision.chat.client.LangChainChatService;
import com.supervision.chat.client.dto.DeleteFileDTO;
import com.supervision.chat.client.dto.LangChainChatRes;
import com.supervision.common.utils.StringUtils;
import com.supervision.config.BusinessException;
import com.supervision.minio.domain.MinioFile;
import com.supervision.minio.service.MinioService;
import com.supervision.police.domain.ModelCase;
@ -17,14 +17,15 @@ import com.supervision.police.dto.NoteRecordDTO;
import com.supervision.police.mapper.NoteRecordMapper;
import com.supervision.police.service.ModelCaseService;
import com.supervision.police.service.NoteRecordService;
import com.supervision.utils.DifyApiUtil;
import com.supervision.utils.Document;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.InputStream;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@Service
@ -38,6 +39,8 @@ public class NoteRecordServiceImpl extends ServiceImpl<NoteRecordMapper, NoteRec
private final MinioService minioService;
private final DifyApiUtil difyApiUtil;
@Override
public List<NoteRecordDTO> selectNoteRecordDTOList(NoteRecord noteRecord) {
return super.getBaseMapper().selectNoteRecordDTOList(noteRecord);
@ -103,6 +106,64 @@ public class NoteRecordServiceImpl extends ServiceImpl<NoteRecordMapper, NoteRec
}
@Override
public void uploadRecordFileToDifyKnowledgeBase(String caseId) {
ModelCase modelCase = modelCaseService.getById(caseId);
if (null == modelCase || StrUtil.isEmpty(modelCase.getKnowledgeBaseId())){
log.warn("uploadRecordFileToKnowledgeBase:案件:{}案件或者知识库为空,不进行知识库维护!", caseId);
return;
}
List<Document> documents = difyApiUtil.queryDocuments(modelCase.getKnowledgeBaseId());
Set<String> documentFileIds = documents.stream().map(Document::getFileId).collect(Collectors.toSet());
List<NoteRecord> recordList = this.lambdaQuery().eq(NoteRecord::getCaseId, caseId).list();
// 只上传 doc docx、txt、md、pdf 文件且文件大小不能超过15mb
List<String> allFileIds = recordList.stream().map(NoteRecord::getFileIds).filter(StrUtil::isNotEmpty)
.flatMap(s -> StrUtil.split(s, ",").stream()).collect(Collectors.toList());
List<MinioFile> minioFiles = minioService.listMinioFile(allFileIds);
List<String> recordFileIds = minioFiles.stream().filter(minioFile -> {
boolean currentFileSize = minioFile.getSize() < 15 * 1024 * 1024;
if (!currentFileSize) {
log.warn("文件大小超过15mb不进行知识库维护:{}", minioFile.getFilename());
return false;
}
boolean currentFileType = StrUtil.equalsAny(minioFile.getFileType(), "doc", "docx", "txt", "md", "pdf");
if (!currentFileType) {
log.warn("文件:{}类型非doc、docx、txt、md、pdf不进行知识库维护...", minioFile.getFilename());
return false;
}
return true;
}).map(MinioFile::getId).toList();
Map<String, MinioFile> fileMap = minioFiles.stream().collect(Collectors.toMap(MinioFile::getId, target -> target));
log.info("案件:{},共有:{}个笔录文件,符合上传要求的文件有:{}",modelCase.getCaseName(), allFileIds.size(), recordFileIds.size());
for (String recordId : recordFileIds) {
// 把新增的笔录数据添加到到知识库
if (!documentFileIds.contains(recordId)){
log.info("案件:{},笔录文件:{},添加到知识库...",modelCase.getCaseName(), fileMap.get(recordId).getFilename());
difyApiUtil.createDocumentByFile(modelCase.getKnowledgeBaseId(),recordId);
}
}
if (CollUtil.isNotEmpty(recordFileIds)){
for (Document document : documents) {
String fileId = document.getFileId();
if (StrUtil.isNotEmpty(fileId) && !recordFileIds.contains(fileId)){
// 删除不在笔录文件列表中的知识库
log.info("案件:{},笔录文件:{},从知识库中删除...",modelCase.getCaseName(), document.getName());
difyApiUtil.deleteDocument(modelCase.getKnowledgeBaseId(),document.getId());
}
}
}
log.info("案件:{}上传笔录文件到知识库完成!",modelCase.getCaseName());
}
@Override
public String saveOrUpdRecord(NoteRecord noteRecord) {

@ -1,5 +1,9 @@
package com.supervision.utils;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.lang.Pair;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
@ -14,6 +18,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.classic.methods.HttpDelete;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.entity.mime.HttpMultipartMode;
import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
@ -24,6 +29,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import static com.supervision.common.constant.DifyConstants.*;
@Component
@ -87,7 +96,7 @@ public class DifyApiUtil {
datasetReqVO.setName(name);
datasetReqVO.setDescription(description);
log.info("创建知识库【{}】", datasetReqVO.getName());
StringEntity entity = new StringEntity(new JSONObject(datasetReqVO).toString(), ContentType.APPLICATION_FORM_URLENCODED);
StringEntity entity = new StringEntity(new JSONObject(datasetReqVO).toString(), ContentType.APPLICATION_JSON);
httpPost.setEntity(entity);
id = httpClient.execute(httpPost, response -> {
final int status = response.getCode();
@ -155,11 +164,14 @@ public class DifyApiUtil {
log.error("文件不存在!ID:{}", fileId);
return null;
}
String fileName = generateDocumentName(minioFile.getFilename(), fileId);
HttpEntity entity = MultipartEntityBuilder.create()
.setCharset(StandardCharsets.UTF_8)
.setMode(HttpMultipartMode.LEGACY)
// JSON 部分
.addTextBody("data", "{\"indexing_technique\":\"high_quality\",\"process_rule\":{\"mode\":\"automatic\"}}", ContentType.APPLICATION_JSON)
// 文件部分
.addBinaryBody("file", minioService.getObjectInputStream(minioFile), ContentType.DEFAULT_BINARY, minioFile.getFilename())
.addBinaryBody("file", minioService.getObjectInputStream(minioFile), ContentType.DEFAULT_BINARY, fileName)
.build();
httpPost.setEntity(entity);
@ -212,4 +224,87 @@ public class DifyApiUtil {
log.error("删除知识库文档失败!", e);
}
}
public DocumentResult queryDocuments(String datasetId, int page, int size){
cn.hutool.http.HttpRequest request = HttpUtil.createGet(difyUrl+ METHOD_DATASET + "/" + datasetId + METHOD_DOCUMENTS);
request.auth(difyDatasetAuth).form("page",page,"limit",size);
try (cn.hutool.http.HttpResponse execute = request.execute()){
String body = execute.body();
if (200 != execute.getStatus()){
log.error("queryDocuments:请求知识库文件列表接口出错error:{}",body);
throw new RuntimeException("queryDocuments:请求知识库文件列表接口出错error:"+body);
}
DocumentResult documentResult = new Gson().fromJson(body, DocumentResult.class);
if (CollUtil.isNotEmpty(documentResult.getData())){
documentResult.getData().forEach(document -> document.setFileId(decodeDocumentName(document.getName()).getValue()));
}
return documentResult;
}
}
public List<Document> queryDocuments(String datasetId){
DocumentResult documentResult = queryDocuments(datasetId, 1, 100);
List<Document> documents = new ArrayList<>(documentResult.getData());
while (documentResult.isHas_more()){
documentResult = queryDocuments(datasetId, documentResult.getPage()+1, 100);
documents.addAll(documentResult.getData());
}
return documents;
}
/**
*
* @param fileName
* @param fileId id
* @return + "_" + ID +"." +
*/
public String generateDocumentName(String fileName, String fileId) {
String[] split = fileName.split("\\.");
List<String> nameTrunk = new ArrayList<>();
for (int i = 0; i < split.length-1; i++) {
nameTrunk.add(split[i]);
}
String documentName = StrUtil.join(".", nameTrunk) + "_" + fileId;
if (split.length > 1) {
documentName = documentName + "." + split[split.length -1];
}
return documentName;
}
/**
*
* @param documentName
* @return keyvalueid
*/
public Pair<String,String> decodeDocumentName(String documentName){
if (StrUtil.isEmpty(documentName)){
return Pair.of(null,null);
}
String[] firstSplit = documentName.split("\\.");
List<String> nameTrunk = new ArrayList<>();
for (int i = 0; i < firstSplit.length -1; i++) {
nameTrunk.add(firstSplit[i]);
}
String name = StrUtil.join(".", nameTrunk);
String[] secondSplit = name.split("_");
nameTrunk = new ArrayList<>();
for (int i = 0; i < secondSplit.length -1; i++) {
nameTrunk.add(secondSplit[i]);
}
String completeName = StrUtil.join("_", nameTrunk) + "." + firstSplit[firstSplit.length-1];
String fileId = secondSplit[secondSplit.length-1];
return Pair.of(completeName,fileId);
}
}

@ -0,0 +1,13 @@
package com.supervision.utils;
import lombok.Data;
@Data
public class Document {
private String id;
private String name;
private String fileId;
}

@ -0,0 +1,19 @@
package com.supervision.utils;
import lombok.Data;
import java.util.List;
@Data
public class DocumentResult {
private List<Document> data;
private boolean has_more;
private int limit;
private int total;
private int page;
}

@ -2,10 +2,12 @@ package com.supervision.demo;
import com.supervision.police.vo.dify.ChatReqVO;
import com.supervision.utils.DifyApiUtil;
import com.supervision.utils.Document;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.util.List;
import java.util.Map;
@SpringBootTest
@ -42,4 +44,11 @@ public class DifyTest {
chatReqVO.setInputs(Map.of("dataset_id", "13c60b8c-341f-43ea-b3cc-5289a518abd9"));
difyApiUtil.chat(chatReqVO);
}
@Test
public void testQueryDocuments() {
List<Document> documents = difyApiUtil.queryDocuments("d6c3e9fa-05a3-4d10-b482-d2797d7eee25");
System.out.println(documents.size());
}
}

@ -15,12 +15,10 @@ import com.supervision.demo.controller.ExampleChatController;
import com.supervision.police.domain.ModelRecordType;
import com.supervision.police.domain.NotePrompt;
import com.supervision.police.domain.NoteRecordSplit;
import com.supervision.police.dto.EvidenceDirectoryDTO;
import com.supervision.police.dto.NotePromptExtractAttributesDto;
import com.supervision.police.dto.RetrieveReqDTO;
import com.supervision.police.dto.RetrieveResDTO;
import com.supervision.police.service.*;
import com.supervision.police.vo.GraphDebugReqVO;
import com.supervision.thread.RecordSplitClassifyTask;
import lombok.AllArgsConstructor;
import lombok.Data;
@ -417,12 +415,16 @@ public class FuHsiApplicationTests {
@Autowired
ModelIndexService modelIndexService;
@Autowired
ModelCaseService modelCaseService;
@Test
public void initCaseKnowledgeBaseTest(){
modelCaseService.initCaseKnowledgeBase("1823935118734643202");
}
@Test
public void aaa(){
GraphDebugReqVO graphDebugReqVO = new GraphDebugReqVO();
graphDebugReqVO.setQueryLang("MATCH (m:`行为人`), (n:`他人`) where m.name=$lawActor and m.caseId = n.caseId =$caseId and m.picType = n.picType = '1' OPTIONAL MATCH (m)-[r:`签订先行合同`]->(n) RETURN id(m) as startId, id(n) as endId, id(r) as relId, m.recordId as recordId, n.recordSplitId as recordSplitId");
graphDebugReqVO.setCaseId("1835994976247951362");
modelIndexService.graphDebug(graphDebugReqVO);
public void migrateRecordKnowledgeBaseTest(){
modelCaseService.migrateRecordKnowledgeBase();
}
}

Loading…
Cancel
Save