Yaxin
/
fu-hsi-service
31
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev_1.0.0' into dev_1.0.0

Browse Source
topo_dev
xueqingkun 9 months ago
parent c827dbdb53 dbab5bcc5a
commit bbe23139a0
10 changed files with 259 additions and 53 deletions
Show Stats Download Patch File Download Diff File

37
src/main/java/com/supervision/chat/UploadParamEnum.java
Unescape Escape View File

@ -0,0 +1,37 @@
package com.supervision.chat;
import org.springframework.web.bind.annotation.RequestPart;
public enum UploadParamEnum {
to_vector_store(null, true, null),
override(null, false, null),
not_refresh_vs_cache(null, false, null),
chunk_size(null, null, 250),
chunk_overlap(null, null, 50),
zh_title_enhance(null, false, null),
docs("{\"test.txt\":[{\"page_content\":\"custom doc\",\"metadata\":{},\"type\":\"Document\"}]}", null, null);
private final String strValue;
private final Boolean booleanValue;
private final Integer intValue;
UploadParamEnum(String strValue, Boolean booleanValue, Integer intValue) {
this.strValue = strValue;
this.booleanValue = booleanValue;
this.intValue = intValue;
}
public String getStrValue() {
return strValue;
}
public Boolean getBooleanValue() {
return booleanValue;
}
public Integer getIntValue() {
return intValue;
}
}

75
src/main/java/com/supervision/chat/client/CustomMultipartFile.java
Unescape Escape View File

@ -0,0 +1,75 @@
package com.supervision.chat.client;
import org.springframework.lang.NonNull;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
import org.springframework.util.FileCopyUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
public class CustomMultipartFile implements MultipartFile {
private final String name;
private final String originalFilename;
@Nullable
private final String contentType;
private final byte[] content;
public CustomMultipartFile(String name, @Nullable byte[] content) {
this(name, name, (String)null, (byte[])content);
}
public CustomMultipartFile(String name, InputStream contentStream) throws IOException {
this(name, name, (String)null, (byte[]) FileCopyUtils.copyToByteArray(contentStream));
}
public CustomMultipartFile(String name, @Nullable String originalFilename, @Nullable String contentType, @Nullable byte[] content) {
Assert.hasLength(name, "Name must not be empty");
this.name = name;
this.originalFilename = originalFilename != null ? originalFilename : "";
this.contentType = contentType;
this.content = content != null ? content : new byte[0];
}
public CustomMultipartFile(String name, @Nullable String originalFilename, @Nullable String contentType, InputStream contentStream) throws IOException {
this(name, originalFilename, contentType, FileCopyUtils.copyToByteArray(contentStream));
}
public String getName() {
return this.name;
}
@NonNull
public String getOriginalFilename() {
return this.originalFilename;
}
@Nullable
public String getContentType() {
return this.contentType;
}
public boolean isEmpty() {
return this.content.length == 0;
}
public long getSize() {
return (long)this.content.length;
}
public byte[] getBytes() throws IOException {
return this.content;
}
public InputStream getInputStream() throws IOException {
return new ByteArrayInputStream(this.content);
}
public void transferTo(File dest) throws IOException, IllegalStateException {
FileCopyUtils.copy(this.content, dest);
}
}

50
src/main/java/com/supervision/chat/client/LangChainChatService.java
Unescape Escape View File

@ -1,6 +1,7 @@
package com.supervision.chat.client;
import com.supervision.chat.client.dto.CreateBaseDTO;
import com.supervision.chat.client.dto.DeleteFileDTO;
import com.supervision.chat.client.dto.LangChainChatRes;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
@ -14,20 +15,47 @@ import org.springframework.web.service.annotation.PostExchange;
@HttpExchange
public interface LangChainChatService {
/**
* 创建知识库
* @param createBaseDTO 知识库对象
* @return 结果
*/
@PostExchange(url = "create_knowledge_base", contentType = MediaType.APPLICATION_JSON_VALUE)
LangChainChatRes chat(@RequestBody CreateBaseDTO createBaseDTO);
LangChainChatRes createBase(@RequestBody CreateBaseDTO createBaseDTO);
/**
* 上传文档接口
* @param knowledge_base_name 需要上传的知识库库名
* @param files 文件,multipartFile
* @param text_splitter_type 问讯笔录
* @param to_vector_store true 固定值
* @param override false 固定值
* @param not_refresh_vs_cache false 固定值
* @param chunk_size 250 固定值
* @param chunk_overlap 50 固定值
* @param zh_title_enhance false 固定值
* @param docs {"test.txt":[{"page_content":"custom doc","metadata":{},"type":"Document"}]} 固定值
* @return 调用的结果
*/
@PostExchange(url = "upload_docs", contentType = MediaType.MULTIPART_FORM_DATA_VALUE)
void uploadFile(@RequestPart String knowledge_base_name,
@RequestPart Resource files,
@RequestPart String to_vector_store,
@RequestPart String override,
@RequestPart String not_refresh_vs_cache,
@RequestPart Integer chunk_size,
@RequestPart Integer chunk_overlap,
@RequestPart String zh_title_enhance,
@RequestPart String text_splitter_type,
@RequestPart String docs);
LangChainChatRes uploadFile(@RequestPart String knowledge_base_name,
@RequestPart MultipartFile files,
@RequestPart String text_splitter_type,
@RequestPart Boolean to_vector_store,
@RequestPart Boolean override,
@RequestPart Boolean not_refresh_vs_cache,
@RequestPart Integer chunk_size,
@RequestPart Integer chunk_overlap,
@RequestPart Boolean zh_title_enhance,
@RequestPart String docs);
/**
* 删除文件
* @param deleteFileDTO 删除的对象
* @return 返回结果
*/
@PostExchange(url = "delete_docs", contentType = MediaType.APPLICATION_JSON_VALUE)
LangChainChatRes deleteFile(@RequestBody DeleteFileDTO deleteFileDTO);
}

32
src/main/java/com/supervision/chat/client/LangChainChatServiceImpl.java
Unescape Escape View File

@ -1,32 +0,0 @@
package com.supervision.chat.client;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONUtil;
import com.supervision.chat.client.dto.CreateBaseDTO;
import com.supervision.chat.client.dto.LangChainChatRes;
import org.apache.hc.client5.http.classic.HttpClient;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpResponse;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.IOException;
@Service
public class LangChainChatServiceImpl{
@Value("${langChain-chat.url}")
private String LangChainChatClientUrl;
public LangChainChatRes chat(CreateBaseDTO createBaseDTO) {
String post = HttpUtil.post(LangChainChatClientUrl + "create_knowledge_base", JSONUtil.toJsonStr(createBaseDTO));
if (JSONUtil.isTypeJSON(post)) {
return JSONUtil.toBean(post, LangChainChatRes.class);
}
return null;
}
}

29
src/main/java/com/supervision/chat/client/dto/DeleteFileDTO.java
Unescape Escape View File

@ -0,0 +1,29 @@
package com.supervision.chat.client.dto;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class DeleteFileDTO {
private String knowledge_base_name;
private List<String> file_names;
private Boolean delete_content = false;
private Boolean not_refresh_vs_cache = false;
public static DeleteFileDTO create(String knowledge_base_name, String file_name) {
DeleteFileDTO deleteFileDTO = new DeleteFileDTO();
deleteFileDTO.setKnowledge_base_name(knowledge_base_name);
List<String> file_names = new ArrayList<>();
file_names.add(file_name);
deleteFileDTO.setFile_names(file_names);
return deleteFileDTO;
}
}

2
src/main/java/com/supervision/chat/client/dto/LangChainChatRes.java
Unescape Escape View File

@ -9,5 +9,5 @@ public class LangChainChatRes {
private String msg;
private String data;
private Object data;
}

32
src/main/java/com/supervision/chat/controller/TestController.java
Unescape Escape View File

@ -1,15 +1,17 @@
package com.supervision.chat.controller;
import cn.hutool.json.JSONUtil;
import com.supervision.chat.client.CustomMultipartFile;
import com.supervision.chat.client.LangChainChatService;
import com.supervision.chat.client.dto.CreateBaseDTO;
import com.supervision.chat.client.dto.DeleteFileDTO;
import com.supervision.chat.client.dto.LangChainChatRes;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.service.annotation.PostExchange;
import org.springframework.web.multipart.MultipartFile;
import java.util.concurrent.Executors;
import java.io.IOException;
@Slf4j
@RequestMapping("chat/test/")
@ -20,11 +22,33 @@ public class TestController {
private final LangChainChatService langChainChatClient;
@GetMapping("test")
public void test(){
public void test() {
CreateBaseDTO createBaseDTO = new CreateBaseDTO();
createBaseDTO.setKnowledge_base_name("11111111");
LangChainChatRes chat = langChainChatClient.chat(createBaseDTO);
LangChainChatRes chat = langChainChatClient.createBase(createBaseDTO);
log.info(JSONUtil.toJsonStr(chat));
}
@PostMapping("uploadFile")
public void testUploadFile(@RequestPart("file") MultipartFile file) throws IOException {
CustomMultipartFile mockMultipartFile = new CustomMultipartFile(file.getOriginalFilename(), file.getInputStream());
LangChainChatRes langChainChatRes = langChainChatClient.uploadFile("11111111",
mockMultipartFile,
"问讯笔录",
"true",
"false",
"false",
250,
50,
"false",
"{\"test.txt\":[{\"page_content\":\"custom doc\",\"metadata\":{},\"type\":\"Document\"}]}");
log.info(JSONUtil.toJsonStr(langChainChatRes));
}
@GetMapping("deleteFile")
public void testDeleteFile(String knowledgeBaseName, String fileName) {
LangChainChatRes langChainChatRes = langChainChatClient.deleteFile(DeleteFileDTO.create(knowledgeBaseName, fileName));
log.info(JSONUtil.toJsonStr(langChainChatRes));
}
}

3
src/main/java/com/supervision/demo/controller/ExampleChatController.java
Unescape Escape View File

@ -14,6 +14,7 @@ import com.supervision.utils.RecordRegexUtil;
import com.supervision.utils.WordReadUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.ibatis.annotations.Param;
import org.json.JSONException;
import org.json.JSONObject;
import org.springframework.ai.chat.ChatResponse;
import org.springframework.ai.chat.Generation;
@ -92,7 +93,7 @@ public class ExampleChatController {
""";
@GetMapping("exampleChat")
public void exampleChat() {
public void exampleChat() throws JSONException {
File file = FileUtil.file("E:\\jc\\宁夏\\Fw_裴金禄\\裴金禄第一次.docx");
String context = WordReadUtil.readWord(file.getPath());
List<QARecordNodeDTO> qaList = RecordRegexUtil.recordRegex(context, "裴金禄");

15
src/main/java/com/supervision/police/service/impl/ModelCaseServiceImpl.java
Unescape Escape View File

@ -11,6 +11,9 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.chat.client.LangChainChatService;
import com.supervision.chat.client.dto.CreateBaseDTO;
import com.supervision.chat.client.dto.LangChainChatRes;
import com.supervision.common.domain.R;
import com.supervision.common.enums.ResultStatusEnum;
import com.supervision.common.exception.CustomException;
@ -57,6 +60,8 @@ public class ModelCaseServiceImpl extends ServiceImpl<ModelCaseMapper, ModelCase
private final CaseStatusManageService caseStatusManageService;
private final LangChainChatService langChainChatService;
/**
* 查询列表
*
@ -151,9 +156,16 @@ public class ModelCaseServiceImpl extends ServiceImpl<ModelCaseMapper, ModelCase
Long num = modelCaseMapper.selectCount(null);
modelCase.setIndexNum(Integer.parseInt(num.toString()) + 1);
i = modelCaseMapper.insert(modelCase);
// 这里需要调用知识库的接口,去保存知识库
CreateBaseDTO createBaseDTO = new CreateBaseDTO();
createBaseDTO.setKnowledge_base_name(modelCase.getCaseNo());
LangChainChatRes chat = langChainChatService.createBase(createBaseDTO);
log.info("创建知识库:{}", chat);
if (200 != chat.getCode()) {
throw new BusinessException("保存知识库失败");
}
}
if (i > 0) {
// TODO 这里需要调用知识库的接口,去保存知识库
return R.okMsg("保存成功");
} else {
return R.fail("保存失败");
@ -161,7 +173,6 @@ public class ModelCaseServiceImpl extends ServiceImpl<ModelCaseMapper, ModelCase
}
@Override
public R<?> del(String id) {
ModelCase modelCase = modelCaseMapper.selectById(id);

37
src/main/java/com/supervision/police/service/impl/NoteRecordSplitServiceImpl.java
Unescape Escape View File

@ -6,9 +6,16 @@ import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.chat.UploadParamEnum;
import com.supervision.chat.client.CustomMultipartFile;
import com.supervision.chat.client.LangChainChatService;
import com.supervision.chat.client.dto.DeleteFileDTO;
import com.supervision.chat.client.dto.LangChainChatRes;
import com.supervision.common.utils.IPages;
import com.supervision.common.utils.ListUtils;
import com.supervision.common.utils.StringUtils;
import com.supervision.config.BusinessException;
import com.supervision.demo.dto.QARecordNodeDTO;
import com.supervision.minio.domain.MinioFile;
import com.supervision.minio.mapper.MinioFileMapper;
import com.supervision.minio.service.MinioService;
@ -16,10 +23,9 @@ import com.supervision.neo4j.service.Neo4jService;
import com.supervision.police.domain.*;
import com.supervision.police.dto.NoteRecordDTO;
import com.supervision.police.dto.NoteRecordDetailDTO;
import com.supervision.police.mapper.NoteRecordSplitMapper;
import com.supervision.police.mapper.NoteRecordMapper;
import com.supervision.police.mapper.NoteRecordSplitMapper;
import com.supervision.police.service.*;
import com.supervision.demo.dto.QARecordNodeDTO;
import com.supervision.utils.RecordRegexUtil;
import com.supervision.utils.WordReadUtil;
import lombok.RequiredArgsConstructor;
@ -64,6 +70,8 @@ public class NoteRecordSplitServiceImpl extends ServiceImpl<NoteRecordSplitMappe
private final Neo4jService neo4jService;
private final LangChainChatService langChainChatService;
@Override
@Transactional(transactionManager = "dataSourceTransactionManager", rollbackFor = Exception.class)
@ -108,6 +116,22 @@ public class NoteRecordSplitServiceImpl extends ServiceImpl<NoteRecordSplitMappe
log.error("从minio中获取文件失败:{}", e.getMessage());
continue;
}
// 这里需要把文件传输,传输到知识库中去
CustomMultipartFile mockMultipartFile = new CustomMultipartFile(minioFile.getFilename(), inputStream);
LangChainChatRes langChainChatRes = langChainChatService.uploadFile("11111111",
mockMultipartFile,
"问讯笔录",
UploadParamEnum.to_vector_store.getBooleanValue(),
UploadParamEnum.override.getBooleanValue(),
UploadParamEnum.not_refresh_vs_cache.getBooleanValue(),
UploadParamEnum.chunk_size.getIntValue(),
UploadParamEnum.chunk_overlap.getIntValue(),
UploadParamEnum.zh_title_enhance.getBooleanValue(),
UploadParamEnum.docs.getStrValue());
if (!langChainChatRes.getCode().equals(200)) {
log.error("上传文件失败:{}", langChainChatRes.getMsg());
throw new RuntimeException("上传文件失败");
}
String context = WordReadUtil.readWord(inputStream);
List<QARecordNodeDTO> qaList = RecordRegexUtil.recordRegex(context, record.getName());
List<NoteRecordSplit> splitList = new ArrayList<>();
@ -254,6 +278,7 @@ public class NoteRecordSplitServiceImpl extends ServiceImpl<NoteRecordSplitMappe
}
@Override
@Transactional(transactionManager = "dataSourceTransactionManager", rollbackFor = Exception.class, noRollbackFor = BusinessException.class)
public void delRecords(String id) {
NoteRecord noteRecord = noteRecordMapper.selectById(id);
@ -262,12 +287,20 @@ public class NoteRecordSplitServiceImpl extends ServiceImpl<NoteRecordSplitMappe
// noteRecordMapper.updateById(noteRecord);
String fileIds = noteRecord.getFileIds();
if (StringUtils.isNotEmpty(fileIds)) {
//删除文件
for (String fileId : fileIds.split(",")) {
MinioFile minioFile = minioFileMapper.selectById(fileId);
// 删除文件
LangChainChatRes langChainChatRes = langChainChatService.deleteFile(DeleteFileDTO.create(noteRecord.getCaseId(), minioFile.getFilename()));
if (!langChainChatRes.getCode().equals(200)) {
throw new BusinessException("删除文件:" + minioFile.getFilename() + " 失败!");
}
minioFile.setDataStatus(StringUtils.getUUID());
minioFileMapper.updateById(minioFile);
minioService.delFile(fileId);
}
}
// 去掉逻辑删除,直接删除

Write Preview
Loading…
Cancel
Save