From 86cc6ab8c9395ed7441e48b5b38c16514b9c1329 Mon Sep 17 00:00:00 2001 From: "DESKTOP-DDTUS3E\\yaxin" Date: Fri, 18 Oct 2024 17:25:41 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E7=A4=BA=E8=AF=8D=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E6=94=B9=E9=80=A0=EF=BC=8Cminio=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E4=B8=B4=E6=97=B6=E6=96=87=E4=BB=B6=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=94=B9=E9=80=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../minio/client/MinioTemplate.java | 18 ++- .../minio/service/impl/MinioServiceImpl.java | 12 +- .../police/controller/PromptController.java | 12 +- .../supervision/police/dto/NotePromptDTO.java | 2 + .../police/service/NotePromptService.java | 2 + .../service/impl/LLMExtractServiceImpl.java | 1 + .../service/impl/NotePromptServiceImpl.java | 103 ++++++++++++++++-- 7 files changed, 120 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/supervision/minio/client/MinioTemplate.java b/src/main/java/com/supervision/minio/client/MinioTemplate.java index e07c8a9..0b2edc3 100644 --- a/src/main/java/com/supervision/minio/client/MinioTemplate.java +++ b/src/main/java/com/supervision/minio/client/MinioTemplate.java @@ -3,16 +3,16 @@ package com.supervision.minio.client; import cn.hutool.core.io.IoUtil; import com.supervision.minio.exception.SysObsException; import io.minio.*; -import io.minio.errors.*; +import io.minio.errors.MinioException; import io.minio.messages.DeleteError; import io.minio.messages.DeleteObject; import jakarta.annotation.PostConstruct; +import jakarta.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import java.security.InvalidKeyException; @@ -36,7 +36,6 @@ public class MinioTemplate { @Value("${minio.bucketName}") private String bucketName; - /** * 上传对象 * @@ -46,12 +45,10 @@ public class MinioTemplate { * @param stream 流 * @param objectSize 大小 * @param contentType contentType - * @return void - * @author qmy - * @version 1.0.0 2021/6/30 9:11 - * @since JDK1.8 + * @param tags 标签 + * @throws SysObsException 异常 */ - public void putObject(String bucketName, String objectName, Map userMetadata, InputStream stream, long objectSize, String contentType) { + public void putObject(String bucketName, String objectName, Map userMetadata, InputStream stream, long objectSize, String contentType, Map tags) { try { minioClient.putObject(PutObjectArgs.builder() .bucket(bucketName) @@ -59,6 +56,7 @@ public class MinioTemplate { .userMetadata(userMetadata) .contentType(contentType == null ? "application/octet-stream" : contentType) .stream(stream, objectSize, -1) + .tags(tags) .build()); } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) { e.printStackTrace(); @@ -121,7 +119,7 @@ public class MinioTemplate { public GetObjectResponse getInputStreamObject(String bucketName, String objectName) { GetObjectResponse inputStream; try { - inputStream = minioClient.getObject( + inputStream = minioClient.getObject( GetObjectArgs.builder().bucket(bucketName).object(objectName).build()); return inputStream; } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) { @@ -227,7 +225,7 @@ public class MinioTemplate { if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { log.info("未找到bucket:{},自动建立", bucketName); minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build()); - }else { + } else { log.info("bucket:{}已存在", bucketName); } } diff --git a/src/main/java/com/supervision/minio/service/impl/MinioServiceImpl.java b/src/main/java/com/supervision/minio/service/impl/MinioServiceImpl.java index e84949a..2a36eb2 100644 --- a/src/main/java/com/supervision/minio/service/impl/MinioServiceImpl.java +++ b/src/main/java/com/supervision/minio/service/impl/MinioServiceImpl.java @@ -8,12 +8,12 @@ import com.supervision.minio.client.MinioTemplate; import com.supervision.minio.domain.MinioFile; import com.supervision.minio.mapper.MinioFileMapper; import com.supervision.minio.service.MinioService; +import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import java.util.*; @@ -45,7 +45,11 @@ public class MinioServiceImpl implements MinioService { minioFile.setSize(in.available()); int i = minioFileMapper.insert(minioFile); if (i > 0) { - minioTemplate.putObject(bucketName, temp ? "temp-" + minioFile.getId() : minioFile.getId(), null, in, in.available(), file.getContentType()); + Map tags = new HashMap<>(); + if (temp) { + tags.put("type", "temp"); + } + minioTemplate.putObject(bucketName, minioFile.getId(), null, in, in.available(), file.getContentType(), tags); return minioFile.getId(); } else { return null; @@ -53,7 +57,7 @@ public class MinioServiceImpl implements MinioService { } @Override - public void downloadFile(String fileId, HttpServletResponse response){ + public void downloadFile(String fileId, HttpServletResponse response) { MinioFile minioFile = minioFileMapper.selectById(fileId); String fileType = minioFile.getFileType(); minioTemplate.getObject(bucketName, fileId, fileType, response); @@ -89,7 +93,7 @@ public class MinioServiceImpl implements MinioService { @Override public List listMinioFile(List fileIdList) { - if (CollUtil.isEmpty(fileIdList)){ + if (CollUtil.isEmpty(fileIdList)) { return new ArrayList<>(1); } return minioFileMapper.selectBatchIds(fileIdList); diff --git a/src/main/java/com/supervision/police/controller/PromptController.java b/src/main/java/com/supervision/police/controller/PromptController.java index abcac03..86bca62 100644 --- a/src/main/java/com/supervision/police/controller/PromptController.java +++ b/src/main/java/com/supervision/police/controller/PromptController.java @@ -18,10 +18,8 @@ import org.springframework.web.bind.annotation.*; @RequiredArgsConstructor public class PromptController { - private final NotePromptService promptService; - /** * 查询提示词 * @@ -31,9 +29,15 @@ public class PromptController { @PostMapping("/list") @Operation(summary = "查询提示词列表") public R> listPrompt(@RequestBody NotePrompt notePrompt, - @RequestParam(required = false, defaultValue = "1") Integer page, - @RequestParam(required = false, defaultValue = "20") Integer size) { + @RequestParam(required = false, defaultValue = "1") Integer page, + @RequestParam(required = false, defaultValue = "20") Integer size) { IPage notePrompts = promptService.listPrompt(page, size, notePrompt); return R.ok(notePrompts); } + + @PostMapping("/promptDebugging") + @Operation(summary = "提示词调试") + public R promptDebugging(@RequestBody NotePromptDTO notePromptDTO) { + return R.ok(promptService.promptDebugging(notePromptDTO)); + } } diff --git a/src/main/java/com/supervision/police/dto/NotePromptDTO.java b/src/main/java/com/supervision/police/dto/NotePromptDTO.java index 9b51081..f7bebf1 100644 --- a/src/main/java/com/supervision/police/dto/NotePromptDTO.java +++ b/src/main/java/com/supervision/police/dto/NotePromptDTO.java @@ -8,4 +8,6 @@ import lombok.EqualsAndHashCode; @Data public class NotePromptDTO extends NotePrompt { private Integer matchNum; + private String fileId; + private String text; } diff --git a/src/main/java/com/supervision/police/service/NotePromptService.java b/src/main/java/com/supervision/police/service/NotePromptService.java index 8522fb3..423b46f 100644 --- a/src/main/java/com/supervision/police/service/NotePromptService.java +++ b/src/main/java/com/supervision/police/service/NotePromptService.java @@ -13,4 +13,6 @@ public interface NotePromptService extends IService { IPage listPrompt(int page, int size, NotePrompt notePrompt); + + String promptDebugging(NotePromptDTO notePromptDTO); } diff --git a/src/main/java/com/supervision/police/service/impl/LLMExtractServiceImpl.java b/src/main/java/com/supervision/police/service/impl/LLMExtractServiceImpl.java index ec457e5..3eef142 100644 --- a/src/main/java/com/supervision/police/service/impl/LLMExtractServiceImpl.java +++ b/src/main/java/com/supervision/police/service/impl/LLMExtractServiceImpl.java @@ -150,6 +150,7 @@ public class LLMExtractServiceImpl implements LLMExtractService { ChatResponse call = ollamaChatClient.call(new Prompt(new UserMessage(prompt))); String content = call.getResult().getOutput().getContent(); JSONObject jsonObject = new JSONObject(content); + log.info("属性提取结果: {}", jsonObject); List extractAttributesList = LLMExtractDto.getExtractAttributes(); extractAttributesMap.forEach((key, value) -> { String attrValue = jsonObject.getString(key); diff --git a/src/main/java/com/supervision/police/service/impl/NotePromptServiceImpl.java b/src/main/java/com/supervision/police/service/impl/NotePromptServiceImpl.java index c740e9b..475e7b0 100644 --- a/src/main/java/com/supervision/police/service/impl/NotePromptServiceImpl.java +++ b/src/main/java/com/supervision/police/service/impl/NotePromptServiceImpl.java @@ -5,25 +5,33 @@ import cn.hutool.core.util.StrUtil; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import com.supervision.police.domain.ModelRecordType; -import com.supervision.police.domain.NotePrompt; -import com.supervision.police.domain.NotePromptTypeRel; -import com.supervision.police.domain.NoteRecordSplit; +import com.supervision.common.constant.NotePromptConstants; +import com.supervision.common.utils.StringUtils; +import com.supervision.demo.dto.QARecordNodeDTO; +import com.supervision.minio.domain.MinioFile; +import com.supervision.minio.service.MinioService; +import com.supervision.police.domain.*; +import com.supervision.police.dto.LLMExtractDto; import com.supervision.police.dto.NotePromptDTO; import com.supervision.police.mapper.NotePromptMapper; -import com.supervision.police.service.ModelRecordTypeService; -import com.supervision.police.service.NotePromptService; -import com.supervision.police.service.NotePromptTypeRelService; -import com.supervision.police.service.NoteRecordSplitService; +import com.supervision.police.service.*; +import com.supervision.utils.RecordRegexUtil; +import com.supervision.utils.WordReadUtil; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.ChatClient; +import org.springframework.ai.chat.ChatResponse; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.prompt.Prompt; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.*; import java.util.stream.Collectors; @Slf4j @@ -40,6 +48,12 @@ public class NotePromptServiceImpl extends ServiceImpl listPromptBySplitId(String recordSplitId) { @@ -96,4 +110,69 @@ public class NotePromptServiceImpl extends ServiceImpl listPrompt(int page, int size, NotePrompt notePrompt) { return notePromptMapper.selectNotePromptWithMatchNum(new Page<>(page, size), notePrompt); } + + @Override + public String promptDebugging(NotePromptDTO notePromptDTO) { + String result = ""; + String text = notePromptDTO.getText(); + String fileId = notePromptDTO.getFileId(); + if (StringUtils.isNotEmpty(fileId)) { + MinioFile minioFile = minioService.getMinioFile(fileId); + if (minioFile != null) { + if (minioFile.getFilename().endsWith(".doc") || minioFile.getFilename().endsWith(".docx")) { + log.info("当前文件名为：{}，当做word文件处理...", minioFile.getFilename()); + text = WordReadUtil.readWord(minioService.getObjectInputStream(minioFile)); + } else if (minioFile.getFilename().endsWith(".txt")) { + log.info("当前文件名为：{}，当做txt文件处理...", minioFile.getFilename()); + InputStream inputStream = minioService.getObjectInputStream(minioFile); + try (inputStream) { + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + text = reader.lines().collect(Collectors.joining("\n")); + } catch (IOException e) { + log.error("读取文件内容失败", e); + } + } + } + } + if (StringUtils.isEmpty(text)) { + log.info("未上传文件且调试文本为空..."); + return result; + } + String type = notePromptDTO.getType(); + if (NotePromptConstants.TYPE_GRAPH_REASONING.equals(type)) { + List qaList = RecordRegexUtil.recordRegex(text, ""); + log.info("拆分问答对:{}", qaList.size()); + if (qaList.isEmpty()) { + return "未找到问答对"; + } + QARecordNodeDTO qaRecordNodeDTO = qaList.get(0); + HashMap paramMap = new HashMap<>(); + paramMap.put("headEntityType", notePromptDTO.getStartEntityType()); + paramMap.put("relation", notePromptDTO.getRelType()); + paramMap.put("tailEntityType", notePromptDTO.getEndEntityType()); + paramMap.put("question", qaRecordNodeDTO.getQuestion()); + paramMap.put("answer", qaRecordNodeDTO.getAnswer()); + paramMap.put("requirement", ""); + log.info("开始尝试提取三元组：{}-{}-{}", notePromptDTO.getStartEntityType(), notePromptDTO.getRelType(), notePromptDTO.getEndEntityType()); + String format = StrUtil.format(notePromptDTO.getPrompt(), paramMap); + log.info("提示词内容:{}", format); + ChatResponse call = chatClient.call(new Prompt(new UserMessage(format))); + String content = call.getResult().getOutput().getContent(); + log.info("三元组提取结果:{}", content); + result = content; + } else if (NotePromptConstants.TYPE_STRUCTURAL_REASONING.equals(type)) { + LLMExtractDto llmExtractDto = new LLMExtractDto(); + llmExtractDto.setText(text); + llmExtractDto.setPrompt(notePromptDTO.getPrompt()); + llmExtractDto.setExtractAttributes(notePromptDTO.getExtractAttributes()); + List llmExtractDtos = llmExtractService.extractAttribute(Collections.singletonList(llmExtractDto)); + if (CollUtil.isNotEmpty(llmExtractDtos)) { + result = llmExtractDtos.get(0).toString(); + } + } else { + log.info("未找到对应的调试类型...【{}】", type); + } + return result; + + } }