From 767d0acf4ca4ce7b389b328731c4f8257c1f3f3e Mon Sep 17 00:00:00 2001 From: "DESKTOP-DDTUS3E\\yaxin" Date: Thu, 21 Nov 2024 17:47:48 +0800 Subject: [PATCH] =?UTF-8?q?DIFY=20API=E5=88=9D=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/constant/DifyConstants.java | 15 ++ .../supervision/police/vo/dify/ChatReqVO.java | 18 ++ .../supervision/police/vo/dify/ChatResVO.java | 16 ++ .../police/vo/dify/DatasetReqVO.java | 13 ++ .../police/vo/dify/DatasetResVO.java | 24 ++ .../supervision/police/vo/dify/Metadata.java | 8 + .../com/supervision/police/vo/dify/Usage.java | 19 ++ .../com/supervision/utils/DifyApiUtil.java | 215 ++++++++++++++++++ .../com/supervision/utils/XxlJobUtil.java | 55 ----- src/main/resources/application-dev.yml | 7 +- src/main/resources/application-prod.yml | 7 +- .../java/com/supervision/demo/DifyTest.java | 45 ++++ .../java/com/supervision/demo/TaskTest.java | 15 +- 13 files changed, 387 insertions(+), 70 deletions(-) create mode 100644 src/main/java/com/supervision/common/constant/DifyConstants.java create mode 100644 src/main/java/com/supervision/police/vo/dify/ChatReqVO.java create mode 100644 src/main/java/com/supervision/police/vo/dify/ChatResVO.java create mode 100644 src/main/java/com/supervision/police/vo/dify/DatasetReqVO.java create mode 100644 src/main/java/com/supervision/police/vo/dify/DatasetResVO.java create mode 100644 src/main/java/com/supervision/police/vo/dify/Metadata.java create mode 100644 src/main/java/com/supervision/police/vo/dify/Usage.java create mode 100644 src/main/java/com/supervision/utils/DifyApiUtil.java delete mode 100644 src/main/java/com/supervision/utils/XxlJobUtil.java create mode 100644 src/test/java/com/supervision/demo/DifyTest.java diff --git a/src/main/java/com/supervision/common/constant/DifyConstants.java b/src/main/java/com/supervision/common/constant/DifyConstants.java new file mode 100644 index 0000000..3adbcd1 --- /dev/null +++ b/src/main/java/com/supervision/common/constant/DifyConstants.java @@ -0,0 +1,15 @@ +package com.supervision.common.constant; + +public class DifyConstants { + public static final String METHOD_DATASET = "/datasets"; + public static final String METHOD_DOCUMENT = "/document"; + public static final String METHOD_DOCUMENTS = "/documents"; + public static final String METHOD_CREATE_BY_FILE = "/create-by-file"; + public static final String METHOD_CHAT_MESSAGES = "/chat-messages"; + public static final String DATASET_INDEXING_TECHNIQUE_HIGH_QUALITY = "high_quality"; + public static final String CHAT_RESPONSE_MODE_BLOCKING = "blocking"; + public static final String CHAT_RESPONSE_MODE_STREAMING = "streaming"; + + + +} diff --git a/src/main/java/com/supervision/police/vo/dify/ChatReqVO.java b/src/main/java/com/supervision/police/vo/dify/ChatReqVO.java new file mode 100644 index 0000000..93574dd --- /dev/null +++ b/src/main/java/com/supervision/police/vo/dify/ChatReqVO.java @@ -0,0 +1,18 @@ +package com.supervision.police.vo.dify; + +import lombok.Data; + +import java.util.HashMap; +import java.util.Map; + +import static com.supervision.common.constant.DifyConstants.CHAT_RESPONSE_MODE_BLOCKING; +import static com.supervision.common.constant.DifyConstants.CHAT_RESPONSE_MODE_STREAMING; + +@Data +public class ChatReqVO { + private String conversationId; + private String user; + private String query; + private String response_mode = CHAT_RESPONSE_MODE_STREAMING; + private Map inputs = new HashMap<>(); +} diff --git a/src/main/java/com/supervision/police/vo/dify/ChatResVO.java b/src/main/java/com/supervision/police/vo/dify/ChatResVO.java new file mode 100644 index 0000000..7f5b338 --- /dev/null +++ b/src/main/java/com/supervision/police/vo/dify/ChatResVO.java @@ -0,0 +1,16 @@ +package com.supervision.police.vo.dify; + +import lombok.Data; + +@Data +public class ChatResVO { + private String event; + private String task_id; + private String id; + private String message_id; + private String conversation_id; + private String mode; + private String answer; + private Metadata metadata; + private String created_at; +} diff --git a/src/main/java/com/supervision/police/vo/dify/DatasetReqVO.java b/src/main/java/com/supervision/police/vo/dify/DatasetReqVO.java new file mode 100644 index 0000000..2186566 --- /dev/null +++ b/src/main/java/com/supervision/police/vo/dify/DatasetReqVO.java @@ -0,0 +1,13 @@ +package com.supervision.police.vo.dify; + +import lombok.Data; + +import static com.supervision.common.constant.DifyConstants.DATASET_INDEXING_TECHNIQUE_HIGH_QUALITY; + +@Data +public class DatasetReqVO { + private String name; + private String description; + private String indexing_technique = DATASET_INDEXING_TECHNIQUE_HIGH_QUALITY; + +} diff --git a/src/main/java/com/supervision/police/vo/dify/DatasetResVO.java b/src/main/java/com/supervision/police/vo/dify/DatasetResVO.java new file mode 100644 index 0000000..13cbc53 --- /dev/null +++ b/src/main/java/com/supervision/police/vo/dify/DatasetResVO.java @@ -0,0 +1,24 @@ +package com.supervision.police.vo.dify; + +import lombok.Data; + +@Data +public class DatasetResVO { + private String id; + private String name; + private String description; + private String provider; + private String permission; + private String data_source_type; + private String indexing_technique; + private String app_count; + private String document_count; + private String word_count; + private String created_by; + private String created_at; + private String updated_by; + private String updated_at; + private String embedding_model; + private String embedding_model_provider; + private String embedding_available; +} diff --git a/src/main/java/com/supervision/police/vo/dify/Metadata.java b/src/main/java/com/supervision/police/vo/dify/Metadata.java new file mode 100644 index 0000000..3fce5db --- /dev/null +++ b/src/main/java/com/supervision/police/vo/dify/Metadata.java @@ -0,0 +1,8 @@ +package com.supervision.police.vo.dify; + +import lombok.Data; + +@Data +public class Metadata { + private Usage usage; +} diff --git a/src/main/java/com/supervision/police/vo/dify/Usage.java b/src/main/java/com/supervision/police/vo/dify/Usage.java new file mode 100644 index 0000000..8ba5643 --- /dev/null +++ b/src/main/java/com/supervision/police/vo/dify/Usage.java @@ -0,0 +1,19 @@ +package com.supervision.police.vo.dify; + +import lombok.Data; + +@Data +public class Usage { + private int promptTokens; + private String promptUnitPrice; + private String promptPriceUnit; + private String promptPrice; + private int completionTokens; + private String completionUnitPrice; + private String completionPriceUnit; + private String completionPrice; + private int totalTokens; + private String totalPrice; + private String currency; + private double latency; +} diff --git a/src/main/java/com/supervision/utils/DifyApiUtil.java b/src/main/java/com/supervision/utils/DifyApiUtil.java new file mode 100644 index 0000000..0152909 --- /dev/null +++ b/src/main/java/com/supervision/utils/DifyApiUtil.java @@ -0,0 +1,215 @@ +package com.supervision.utils; + +import cn.hutool.json.JSONObject; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.supervision.minio.domain.MinioFile; +import com.supervision.minio.service.MinioService; +import com.supervision.police.vo.dify.ChatReqVO; +import com.supervision.police.vo.dify.ChatResVO; +import com.supervision.police.vo.dify.DatasetReqVO; +import com.supervision.police.vo.dify.DatasetResVO; +import lombok.extern.slf4j.Slf4j; +import org.apache.hc.client5.http.ClientProtocolException; +import org.apache.hc.client5.http.classic.methods.HttpDelete; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.*; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import static com.supervision.common.constant.DifyConstants.*; + +@Component +@Slf4j +public class DifyApiUtil { + + @Value("${dify.url}") + private String difyUrl; + @Value("${dify.dataset-auth}") + private String difyDatasetAuth; + @Value("${dify.app-auth}") + private String difyAppAuth; + + @Autowired + private MinioService minioService; + + public void chat(ChatReqVO chatReqVO) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost httpPost = new HttpPost(difyUrl + METHOD_CHAT_MESSAGES); + httpPost.setHeader(HttpHeaders.AUTHORIZATION, difyAppAuth); + httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); + log.info("发起对话：{}", chatReqVO); + StringEntity entity = new StringEntity(new JSONObject(chatReqVO).toString(), ContentType.APPLICATION_FORM_URLENCODED); + httpPost.setEntity(entity); + ChatResVO chatResVO1 = httpClient.execute(httpPost, response -> { + final int status = response.getCode(); + if (status >= HttpStatus.SC_SUCCESS && status < HttpStatus.SC_REDIRECTION) { + final HttpEntity responseEntity = response.getEntity(); + try { + String responseStr = EntityUtils.toString(responseEntity); + log.info("responseStr:{}", responseStr); + return new ChatResVO(); + } catch (final ParseException ex) { + throw new ClientProtocolException(ex); + } + } else { + log.info("responseEntity {}", EntityUtils.toString(response.getEntity())); + throw new ClientProtocolException("Unexpected response status: " + status); + } + }); + log.info("chatResVO1:{}", chatResVO1); + } catch (Exception e) { + log.error("create dataset error", e); + } + } + + /** + * 创建知识库 + * + * @param name 知识库名称 + * @param description 知识库描述 + * @return 知识库ID + */ + public String createDataset(String name, String description) { + String id = ""; + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost httpPost = new HttpPost(difyUrl + METHOD_DATASET); + httpPost.setHeader(HttpHeaders.AUTHORIZATION, difyDatasetAuth); + httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "application/json"); + DatasetReqVO datasetReqVO = new DatasetReqVO(); + datasetReqVO.setName(name); + datasetReqVO.setDescription(description); + log.info("创建知识库【{}】", datasetReqVO.getName()); + StringEntity entity = new StringEntity(new JSONObject(datasetReqVO).toString(), ContentType.APPLICATION_FORM_URLENCODED); + httpPost.setEntity(entity); + id = httpClient.execute(httpPost, response -> { + final int status = response.getCode(); + if (status >= HttpStatus.SC_SUCCESS && status < HttpStatus.SC_REDIRECTION) { + final HttpEntity responseEntity = response.getEntity(); + try { + String responseStr = EntityUtils.toString(responseEntity); + Gson gson = new Gson(); + DatasetResVO datasetResVO = gson.fromJson(responseStr, DatasetResVO.class); + log.info("创建知识库成功!ID:【{}】", datasetResVO.getId()); + return datasetResVO.getId(); + } catch (final ParseException ex) { + throw new ClientProtocolException(ex); + } + } else { + log.info("创建知识库失败!{}", EntityUtils.toString(response.getEntity())); + return null; + } + }); + } catch (Exception e) { + log.error("创建知识库失败!", e); + } + return id; + } + + /** + * 删除知识库 + * + * @param id 知识库ID + */ + public void deleteDataset(String id) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpDelete httpDelete = new HttpDelete(difyUrl + METHOD_DATASET + "/" + id); + httpDelete.setHeader(HttpHeaders.AUTHORIZATION, difyDatasetAuth); + log.info("删除知识库【{}】", id); + httpClient.execute(httpDelete, response -> { + final int status = response.getCode(); + if (status == HttpStatus.SC_NO_CONTENT) { + log.info("删除知识库成功!"); + } else { + log.info("删除知识库失败!状态码：{}", status); + } + return null; + }); + } catch (Exception e) { + log.error("删除知识库失败!", e); + } + } + + /** + * 通过文件创建知识库文档 + * + * @param datasetId 知识库ID + * @param fileId 文件ID + * @return 文档ID + */ + public String createDocumentByFile(String datasetId, String fileId) { + String id = ""; + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + log.info("创建知识库文档【{}】", difyUrl + METHOD_DATASET + "/" + datasetId + METHOD_DOCUMENT + METHOD_CREATE_BY_FILE); + HttpPost httpPost = new HttpPost(difyUrl + METHOD_DATASET + "/" + datasetId + METHOD_DOCUMENT + METHOD_CREATE_BY_FILE); + httpPost.setHeader(HttpHeaders.AUTHORIZATION, difyDatasetAuth); + MinioFile minioFile = minioService.getMinioFile(fileId); + if (minioFile == null) { + log.error("文件不存在!ID:{}", fileId); + return null; + } + HttpEntity entity = MultipartEntityBuilder.create() + // JSON 部分 + .addTextBody("data", "{\"indexing_technique\":\"high_quality\",\"process_rule\":{\"mode\":\"automatic\"}}", ContentType.APPLICATION_JSON) + // 文件部分 + .addBinaryBody("file", minioService.getObjectInputStream(minioFile), ContentType.DEFAULT_BINARY, minioFile.getFilename()) + .build(); + + httpPost.setEntity(entity); + id = httpClient.execute(httpPost, response -> { + final int status = response.getCode(); + if (status >= HttpStatus.SC_SUCCESS && status < HttpStatus.SC_REDIRECTION) { + final HttpEntity responseEntity = response.getEntity(); + try { + String responseStr = EntityUtils.toString(responseEntity); + log.info("responseStr:{}", responseStr); + JsonObject jsonObject = JsonParser.parseString(responseStr).getAsJsonObject(); + JsonObject document = jsonObject.getAsJsonObject("document"); + log.info("创建知识库文档成功!ID:{}", document.get("id").getAsString()); + return document.get("id").getAsString(); + } catch (final ParseException ex) { + throw new ClientProtocolException(ex); + } + } else { + log.info("创建知识库文档失败!{}", EntityUtils.toString(response.getEntity())); + return null; + } + }); + } catch (Exception e) { + log.error("创建知识库文档失败!", e); + } + return id; + } + + /** + * 删除知识库文档 + * + * @param datasetId 知识库ID + * @param documentId 文档ID + */ + public void deleteDocument(String datasetId, String documentId) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpDelete httpDelete = new HttpDelete(difyUrl + METHOD_DATASET + "/" + datasetId + METHOD_DOCUMENTS + "/" + documentId); + httpDelete.setHeader(HttpHeaders.AUTHORIZATION, difyDatasetAuth); + log.info("删除知识库文档【{}】", documentId); + httpClient.execute(httpDelete, response -> { + final int status = response.getCode(); + if (status == HttpStatus.SC_SUCCESS) { + log.info("删除知识库文档成功!"); + } else { + log.info("删除知识库文档失败!状态码：{}", status); + } + return null; + }); + } catch (Exception e) { + log.error("删除知识库文档失败!", e); + } + } +} diff --git a/src/main/java/com/supervision/utils/XxlJobUtil.java b/src/main/java/com/supervision/utils/XxlJobUtil.java deleted file mode 100644 index bd1bb37..0000000 --- a/src/main/java/com/supervision/utils/XxlJobUtil.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.supervision.utils; - -import org.apache.hc.client5.http.classic.methods.HttpGet; -import org.apache.hc.client5.http.classic.methods.HttpPost; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.apache.hc.core5.http.io.entity.EntityUtils; -import org.apache.hc.core5.http.io.entity.StringEntity; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @Author: JCccc - * @Date: 2022-6-22 9:51 - * @Description: - */ -@Component -public class XxlJobUtil { - private static String cookie = ""; - @Value("${xxl.job.admin.addresses}") - private String xxlJobAdminAddress; - - public void executeTask(String taskName) throws Exception { - String taskQueryUrl = xxlJobAdminAddress + "/xxl-job-admin/api/jobinfo/load"; - String taskExecuteUrl = xxlJobAdminAddress + "/xxl-job-admin/api/job/trigger"; - - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - // 查询任务 - HttpGet httpGet = new HttpGet(taskQueryUrl + "?jobName=" + taskName); - try (CloseableHttpResponse response = httpClient.execute(httpGet)) { - String jsonResponse = EntityUtils.toString(response.getEntity()); - // 处理任务信息，例如提取 jobId - // 假设 jsonResponse 中有一个字段 "jobId" - String jobId = extractJobId(jsonResponse); - - // 执行任务 - HttpPost httpPost = new HttpPost(taskExecuteUrl); - httpPost.setHeader("Content-Type", "application/json"); - String jsonBody = "{\"jobId\": \"" + jobId + "\"}"; - httpPost.setEntity(new StringEntity(jsonBody)); - try (CloseableHttpResponse executeResponse = httpClient.execute(httpPost)) { - System.out.println("Task executed: " + EntityUtils.toString(executeResponse.getEntity())); - } - } - } - } - - private String extractJobId(String jsonResponse) { - // 解析 JSON 响应并提取 jobId - // 这里需要使用 JSON 解析库，比如 Jackson 或 Gson - // 示例代码略 - return "yourJobId"; // 请替换为实际提取的 jobId - } -} \ No newline at end of file diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index de5f0e0..cc67bf9 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -92,4 +92,9 @@ xxl: ip: port: 9999 logpath: /data/applogs/xxl-job/jobhandler - logretentiondays: 30 \ No newline at end of file + logretentiondays: 30 + +dify: + url: http://192.168.10.137/v1 + dataset-auth: Bearer dataset-PLOwR22cFObxN1AlGM0QdBXT + app-auth: Bearer app-pMR3NUdDtTAcCiyGcohIMjVi diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index e8351d8..ed6d4de 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -92,4 +92,9 @@ xxl: ip: port: 9999 logpath: /data/applogs/xxl-job/jobhandler - logretentiondays: 30 \ No newline at end of file + logretentiondays: 30 + +dify: + url: http://192.168.10.137/v1 + dataset-auth: Bearer dataset-PLOwR22cFObxN1AlGM0QdBXT + app-auth: Bearer app-pMR3NUdDtTAcCiyGcohIMjVi diff --git a/src/test/java/com/supervision/demo/DifyTest.java b/src/test/java/com/supervision/demo/DifyTest.java new file mode 100644 index 0000000..8b60bcd --- /dev/null +++ b/src/test/java/com/supervision/demo/DifyTest.java @@ -0,0 +1,45 @@ +package com.supervision.demo; + +import com.supervision.police.vo.dify.ChatReqVO; +import com.supervision.utils.DifyApiUtil; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.Map; + +@SpringBootTest +public class DifyTest { + @Autowired + private DifyApiUtil difyApiUtil; + + @Test + public void testCreateDataset() { + System.out.println(difyApiUtil.createDataset("张三1", "张三1")); + } + + @Test + public void testDeleteDataset() { + difyApiUtil.deleteDataset("65c3a191-2433-4f79-9fd2-5c4dfaf4264d"); + } + + @Test + public void testCreateDocumentByFile() { + System.out.println(difyApiUtil.createDocumentByFile("92e613e7-7369-46b1-ab12-b3d42dd5b646", "1823953980884635650")); + } + + @Test + public void testDeleteDocument() { + difyApiUtil.deleteDocument("92e613e7-7369-46b1-ab12-b3d42dd5b646", "3f2a3719-1ee8-4cfd-b50c-d469954ee72e"); + } + + @Test + public void testChat() { + System.out.println("Test"); + ChatReqVO chatReqVO = new ChatReqVO(); + chatReqVO.setUser("admin"); + chatReqVO.setQuery("Who are you?"); + chatReqVO.setInputs(Map.of("dataset_id", "13c60b8c-341f-43ea-b3cc-5289a518abd9")); + difyApiUtil.chat(chatReqVO); + } +} diff --git a/src/test/java/com/supervision/demo/TaskTest.java b/src/test/java/com/supervision/demo/TaskTest.java index 0e2fcb2..db1c2d3 100644 --- a/src/test/java/com/supervision/demo/TaskTest.java +++ b/src/test/java/com/supervision/demo/TaskTest.java @@ -1,10 +1,7 @@ package com.supervision.demo; import com.supervision.police.domain.XxlJobInfo; -import com.supervision.police.dto.LLMExtractDto; import com.supervision.police.service.XxlJobService; -import com.supervision.utils.XxlJobUtil; -import com.xxl.job.core.executor.impl.XxlJobSpringExecutor; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -15,12 +12,8 @@ import java.util.List; @Slf4j @SpringBootTest public class TaskTest { - @Autowired - private XxlJobUtil xxlJobUtil; @Autowired private XxlJobService xxlJobService; - @Autowired - private XxlJobSpringExecutor xxlJobSpringExecutor; @Test public void testTask() { @@ -29,15 +22,11 @@ public class TaskTest { XxlJobInfo xxlJobInfo = new XxlJobInfo(); xxlJobInfo.setExecutorHandler("extractAttribute"); List xxlJobInfoList = xxlJobService.pageList(xxlJobInfo); - if(xxlJobInfoList!=null&&!xxlJobInfoList.isEmpty()){ + if (xxlJobInfoList != null && !xxlJobInfoList.isEmpty()) { xxlJobInfo = xxlJobInfoList.get(0); - log.info("xxlJobInfo:{}",xxlJobInfo); + log.info("xxlJobInfo:{}", xxlJobInfo); xxlJobService.executeTaskById(String.valueOf(xxlJobInfo.getId()), null); } -// xxlJobService.executeTask("testTask"); -// Thread.sleep(2000); -// xxlJobUtil.executeTask("testTask"); - } catch (Exception e) { throw new RuntimeException(e); }