Yaxin
/
fu-hsi-service
31
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into dev

Browse Source
topo_dev
xueqingkun 5 months ago
parent 96c7b91ec3 767d0acf4c
commit e22bc18e9e
13 changed files with 387 additions and 70 deletions
Show Stats Download Patch File Download Diff File

15
src/main/java/com/supervision/common/constant/DifyConstants.java
Unescape Escape View File

@ -0,0 +1,15 @@
package com.supervision.common.constant;
public class DifyConstants {
public static final String METHOD_DATASET = "/datasets";
public static final String METHOD_DOCUMENT = "/document";
public static final String METHOD_DOCUMENTS = "/documents";
public static final String METHOD_CREATE_BY_FILE = "/create-by-file";
public static final String METHOD_CHAT_MESSAGES = "/chat-messages";
public static final String DATASET_INDEXING_TECHNIQUE_HIGH_QUALITY = "high_quality";
public static final String CHAT_RESPONSE_MODE_BLOCKING = "blocking";
public static final String CHAT_RESPONSE_MODE_STREAMING = "streaming";
}

18
src/main/java/com/supervision/police/vo/dify/ChatReqVO.java
Unescape Escape View File

@ -0,0 +1,18 @@
package com.supervision.police.vo.dify;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
import static com.supervision.common.constant.DifyConstants.CHAT_RESPONSE_MODE_BLOCKING;
import static com.supervision.common.constant.DifyConstants.CHAT_RESPONSE_MODE_STREAMING;
@Data
public class ChatReqVO {
private String conversationId;
private String user;
private String query;
private String response_mode = CHAT_RESPONSE_MODE_STREAMING;
private Map<String, Object> inputs = new HashMap<>();
}

16
src/main/java/com/supervision/police/vo/dify/ChatResVO.java
Unescape Escape View File

@ -0,0 +1,16 @@
package com.supervision.police.vo.dify;
import lombok.Data;
@Data
public class ChatResVO {
private String event;
private String task_id;
private String id;
private String message_id;
private String conversation_id;
private String mode;
private String answer;
private Metadata metadata;
private String created_at;
}

13
src/main/java/com/supervision/police/vo/dify/DatasetReqVO.java
Unescape Escape View File

@ -0,0 +1,13 @@
package com.supervision.police.vo.dify;
import lombok.Data;
import static com.supervision.common.constant.DifyConstants.DATASET_INDEXING_TECHNIQUE_HIGH_QUALITY;
@Data
public class DatasetReqVO {
private String name;
private String description;
private String indexing_technique = DATASET_INDEXING_TECHNIQUE_HIGH_QUALITY;
}

24
src/main/java/com/supervision/police/vo/dify/DatasetResVO.java
Unescape Escape View File

@ -0,0 +1,24 @@
package com.supervision.police.vo.dify;
import lombok.Data;
@Data
public class DatasetResVO {
private String id;
private String name;
private String description;
private String provider;
private String permission;
private String data_source_type;
private String indexing_technique;
private String app_count;
private String document_count;
private String word_count;
private String created_by;
private String created_at;
private String updated_by;
private String updated_at;
private String embedding_model;
private String embedding_model_provider;
private String embedding_available;
}

8
src/main/java/com/supervision/police/vo/dify/Metadata.java
Unescape Escape View File

@ -0,0 +1,8 @@
package com.supervision.police.vo.dify;
import lombok.Data;
@Data
public class Metadata {
private Usage usage;
}

19
src/main/java/com/supervision/police/vo/dify/Usage.java
Unescape Escape View File

@ -0,0 +1,19 @@
package com.supervision.police.vo.dify;
import lombok.Data;
@Data
public class Usage {
private int promptTokens;
private String promptUnitPrice;
private String promptPriceUnit;
private String promptPrice;
private int completionTokens;
private String completionUnitPrice;
private String completionPriceUnit;
private String completionPrice;
private int totalTokens;
private String totalPrice;
private String currency;
private double latency;
}

215
src/main/java/com/supervision/utils/DifyApiUtil.java
Unescape Escape View File

@ -0,0 +1,215 @@
package com.supervision.utils;
import cn.hutool.json.JSONObject;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.supervision.minio.domain.MinioFile;
import com.supervision.minio.service.MinioService;
import com.supervision.police.vo.dify.ChatReqVO;
import com.supervision.police.vo.dify.ChatResVO;
import com.supervision.police.vo.dify.DatasetReqVO;
import com.supervision.police.vo.dify.DatasetResVO;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.classic.methods.HttpDelete;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.*;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import static com.supervision.common.constant.DifyConstants.*;
@Component
@Slf4j
public class DifyApiUtil {
@Value("${dify.url}")
private String difyUrl;
@Value("${dify.dataset-auth}")
private String difyDatasetAuth;
@Value("${dify.app-auth}")
private String difyAppAuth;
@Autowired
private MinioService minioService;
public void chat(ChatReqVO chatReqVO) {
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
HttpPost httpPost = new HttpPost(difyUrl + METHOD_CHAT_MESSAGES);
httpPost.setHeader(HttpHeaders.AUTHORIZATION, difyAppAuth);
httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "application/json");
log.info("发起对话：{}", chatReqVO);
StringEntity entity = new StringEntity(new JSONObject(chatReqVO).toString(), ContentType.APPLICATION_FORM_URLENCODED);
httpPost.setEntity(entity);
ChatResVO chatResVO1 = httpClient.execute(httpPost, response -> {
final int status = response.getCode();
if (status >= HttpStatus.SC_SUCCESS && status < HttpStatus.SC_REDIRECTION) {
final HttpEntity responseEntity = response.getEntity();
try {
String responseStr = EntityUtils.toString(responseEntity);
log.info("responseStr:{}", responseStr);
return new ChatResVO();
} catch (final ParseException ex) {
throw new ClientProtocolException(ex);
}
} else {
log.info("responseEntity {}", EntityUtils.toString(response.getEntity()));
throw new ClientProtocolException("Unexpected response status: " + status);
}
});
log.info("chatResVO1:{}", chatResVO1);
} catch (Exception e) {
log.error("create dataset error", e);
}
}
/**
* 创建知识库
*
* @param name 知识库名称
* @param description 知识库描述
* @return 知识库ID
*/
public String createDataset(String name, String description) {
String id = "";
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
HttpPost httpPost = new HttpPost(difyUrl + METHOD_DATASET);
httpPost.setHeader(HttpHeaders.AUTHORIZATION, difyDatasetAuth);
httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "application/json");
DatasetReqVO datasetReqVO = new DatasetReqVO();
datasetReqVO.setName(name);
datasetReqVO.setDescription(description);
log.info("创建知识库【{}】", datasetReqVO.getName());
StringEntity entity = new StringEntity(new JSONObject(datasetReqVO).toString(), ContentType.APPLICATION_FORM_URLENCODED);
httpPost.setEntity(entity);
id = httpClient.execute(httpPost, response -> {
final int status = response.getCode();
if (status >= HttpStatus.SC_SUCCESS && status < HttpStatus.SC_REDIRECTION) {
final HttpEntity responseEntity = response.getEntity();
try {
String responseStr = EntityUtils.toString(responseEntity);
Gson gson = new Gson();
DatasetResVO datasetResVO = gson.fromJson(responseStr, DatasetResVO.class);
log.info("创建知识库成功!ID:【{}】", datasetResVO.getId());
return datasetResVO.getId();
} catch (final ParseException ex) {
throw new ClientProtocolException(ex);
}
} else {
log.info("创建知识库失败!{}", EntityUtils.toString(response.getEntity()));
return null;
}
});
} catch (Exception e) {
log.error("创建知识库失败!", e);
}
return id;
}
/**
* 删除知识库
*
* @param id 知识库ID
*/
public void deleteDataset(String id) {
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
HttpDelete httpDelete = new HttpDelete(difyUrl + METHOD_DATASET + "/" + id);
httpDelete.setHeader(HttpHeaders.AUTHORIZATION, difyDatasetAuth);
log.info("删除知识库【{}】", id);
httpClient.execute(httpDelete, response -> {
final int status = response.getCode();
if (status == HttpStatus.SC_NO_CONTENT) {
log.info("删除知识库成功!");
} else {
log.info("删除知识库失败!状态码：{}", status);
}
return null;
});
} catch (Exception e) {
log.error("删除知识库失败!", e);
}
}
/**
* 通过文件创建知识库文档
*
* @param datasetId 知识库ID
* @param fileId 文件ID
* @return 文档ID
*/
public String createDocumentByFile(String datasetId, String fileId) {
String id = "";
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
log.info("创建知识库文档【{}】", difyUrl + METHOD_DATASET + "/" + datasetId + METHOD_DOCUMENT + METHOD_CREATE_BY_FILE);
HttpPost httpPost = new HttpPost(difyUrl + METHOD_DATASET + "/" + datasetId + METHOD_DOCUMENT + METHOD_CREATE_BY_FILE);
httpPost.setHeader(HttpHeaders.AUTHORIZATION, difyDatasetAuth);
MinioFile minioFile = minioService.getMinioFile(fileId);
if (minioFile == null) {
log.error("文件不存在!ID:{}", fileId);
return null;
}
HttpEntity entity = MultipartEntityBuilder.create()
// JSON 部分
.addTextBody("data", "{\"indexing_technique\":\"high_quality\",\"process_rule\":{\"mode\":\"automatic\"}}", ContentType.APPLICATION_JSON)
// 文件部分
.addBinaryBody("file", minioService.getObjectInputStream(minioFile), ContentType.DEFAULT_BINARY, minioFile.getFilename())
.build();
httpPost.setEntity(entity);
id = httpClient.execute(httpPost, response -> {
final int status = response.getCode();
if (status >= HttpStatus.SC_SUCCESS && status < HttpStatus.SC_REDIRECTION) {
final HttpEntity responseEntity = response.getEntity();
try {
String responseStr = EntityUtils.toString(responseEntity);
log.info("responseStr:{}", responseStr);
JsonObject jsonObject = JsonParser.parseString(responseStr).getAsJsonObject();
JsonObject document = jsonObject.getAsJsonObject("document");
log.info("创建知识库文档成功!ID:{}", document.get("id").getAsString());
return document.get("id").getAsString();
} catch (final ParseException ex) {
throw new ClientProtocolException(ex);
}
} else {
log.info("创建知识库文档失败!{}", EntityUtils.toString(response.getEntity()));
return null;
}
});
} catch (Exception e) {
log.error("创建知识库文档失败!", e);
}
return id;
}
/**
* 删除知识库文档
*
* @param datasetId 知识库ID
* @param documentId 文档ID
*/
public void deleteDocument(String datasetId, String documentId) {
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
HttpDelete httpDelete = new HttpDelete(difyUrl + METHOD_DATASET + "/" + datasetId + METHOD_DOCUMENTS + "/" + documentId);
httpDelete.setHeader(HttpHeaders.AUTHORIZATION, difyDatasetAuth);
log.info("删除知识库文档【{}】", documentId);
httpClient.execute(httpDelete, response -> {
final int status = response.getCode();
if (status == HttpStatus.SC_SUCCESS) {
log.info("删除知识库文档成功!");
} else {
log.info("删除知识库文档失败!状态码：{}", status);
}
return null;
});
} catch (Exception e) {
log.error("删除知识库文档失败!", e);
}
}
}

55
src/main/java/com/supervision/utils/XxlJobUtil.java
Unescape Escape View File

@ -1,55 +0,0 @@
package com.supervision.utils;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
/**
* @Author: JCccc
* @Date: 2022-6-22 9:51
* @Description:
*/
@Component
public class XxlJobUtil {
private static String cookie = "";
@Value("${xxl.job.admin.addresses}")
private String xxlJobAdminAddress;
public void executeTask(String taskName) throws Exception {
String taskQueryUrl = xxlJobAdminAddress + "/xxl-job-admin/api/jobinfo/load";
String taskExecuteUrl = xxlJobAdminAddress + "/xxl-job-admin/api/job/trigger";
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
// 查询任务
HttpGet httpGet = new HttpGet(taskQueryUrl + "?jobName=" + taskName);
try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
String jsonResponse = EntityUtils.toString(response.getEntity());
// 处理任务信息，例如提取 jobId
// 假设 jsonResponse 中有一个字段 "jobId"
String jobId = extractJobId(jsonResponse);
// 执行任务
HttpPost httpPost = new HttpPost(taskExecuteUrl);
httpPost.setHeader("Content-Type", "application/json");
String jsonBody = "{\"jobId\": \"" + jobId + "\"}";
httpPost.setEntity(new StringEntity(jsonBody));
try (CloseableHttpResponse executeResponse = httpClient.execute(httpPost)) {
System.out.println("Task executed: " + EntityUtils.toString(executeResponse.getEntity()));
}
}
}
}
private String extractJobId(String jsonResponse) {
// 解析 JSON 响应并提取 jobId
// 这里需要使用 JSON 解析库，比如 Jackson 或 Gson
// 示例代码略
return "yourJobId"; // 请替换为实际提取的 jobId
}
}

7
src/main/resources/application-dev.yml
Unescape Escape View File

@ -92,4 +92,9 @@ xxl:
ip:
port: 9999
logpath: /data/applogs/xxl-job/jobhandler
logretentiondays: 30
logretentiondays: 30
dify:
url: http://192.168.10.137/v1
dataset-auth: Bearer dataset-PLOwR22cFObxN1AlGM0QdBXT
app-auth: Bearer app-pMR3NUdDtTAcCiyGcohIMjVi

7
src/main/resources/application-prod.yml
Unescape Escape View File

@ -92,4 +92,9 @@ xxl:
ip:
port: 9999
logpath: /data/applogs/xxl-job/jobhandler
logretentiondays: 30
logretentiondays: 30
dify:
url: http://192.168.10.137/v1
dataset-auth: Bearer dataset-PLOwR22cFObxN1AlGM0QdBXT
app-auth: Bearer app-pMR3NUdDtTAcCiyGcohIMjVi

45
src/test/java/com/supervision/demo/DifyTest.java
Unescape Escape View File

@ -0,0 +1,45 @@
package com.supervision.demo;
import com.supervision.police.vo.dify.ChatReqVO;
import com.supervision.utils.DifyApiUtil;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.util.Map;
@SpringBootTest
public class DifyTest {
@Autowired
private DifyApiUtil difyApiUtil;
@Test
public void testCreateDataset() {
System.out.println(difyApiUtil.createDataset("张三1", "张三1"));
}
@Test
public void testDeleteDataset() {
difyApiUtil.deleteDataset("65c3a191-2433-4f79-9fd2-5c4dfaf4264d");
}
@Test
public void testCreateDocumentByFile() {
System.out.println(difyApiUtil.createDocumentByFile("92e613e7-7369-46b1-ab12-b3d42dd5b646", "1823953980884635650"));
}
@Test
public void testDeleteDocument() {
difyApiUtil.deleteDocument("92e613e7-7369-46b1-ab12-b3d42dd5b646", "3f2a3719-1ee8-4cfd-b50c-d469954ee72e");
}
@Test
public void testChat() {
System.out.println("Test");
ChatReqVO chatReqVO = new ChatReqVO();
chatReqVO.setUser("admin");
chatReqVO.setQuery("Who are you?");
chatReqVO.setInputs(Map.of("dataset_id", "13c60b8c-341f-43ea-b3cc-5289a518abd9"));
difyApiUtil.chat(chatReqVO);
}
}

15
src/test/java/com/supervision/demo/TaskTest.java
Unescape Escape View File

@ -1,10 +1,7 @@
package com.supervision.demo;
import com.supervision.police.domain.XxlJobInfo;
import com.supervision.police.dto.LLMExtractDto;
import com.supervision.police.service.XxlJobService;
import com.supervision.utils.XxlJobUtil;
import com.xxl.job.core.executor.impl.XxlJobSpringExecutor;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
@ -15,12 +12,8 @@ import java.util.List;
@Slf4j
@SpringBootTest
public class TaskTest {
@Autowired
private XxlJobUtil xxlJobUtil;
@Autowired
private XxlJobService xxlJobService;
@Autowired
private XxlJobSpringExecutor xxlJobSpringExecutor;
@Test
public void testTask() {
@ -29,15 +22,11 @@ public class TaskTest {
XxlJobInfo xxlJobInfo = new XxlJobInfo();
xxlJobInfo.setExecutorHandler("extractAttribute");
List<XxlJobInfo> xxlJobInfoList = xxlJobService.pageList(xxlJobInfo);
if(xxlJobInfoList!=null&&!xxlJobInfoList.isEmpty()){
if (xxlJobInfoList != null && !xxlJobInfoList.isEmpty()) {
xxlJobInfo = xxlJobInfoList.get(0);
log.info("xxlJobInfo:{}",xxlJobInfo);
log.info("xxlJobInfo:{}", xxlJobInfo);
xxlJobService.executeTaskById(String.valueOf(xxlJobInfo.getId()), null);
}
// xxlJobService.executeTask("testTask");
// Thread.sleep(2000);
// xxlJobUtil.executeTask("testTask");
} catch (Exception e) {
throw new RuntimeException(e);
}

Write Preview
Loading…
Cancel
Save