RAG代码提交
liu
8 months ago
parent
c5182c9e78
commit
665ee4af13
@ -0,0 +1,26 @@
|
||||
package som.supervision.knowsub.controller;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import som.supervision.knowsub.service.KnowledgeEtlService;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
@Tag(name = "知识ETL类")
|
||||
@RestController
|
||||
@RequestMapping("etl")
|
||||
@RequiredArgsConstructor
|
||||
public class KnowledgeEtlController {
|
||||
|
||||
private final KnowledgeEtlService knowledgeEtlService;
|
||||
|
||||
@Operation(summary = "对知识进行ETL")
|
||||
@PostMapping("knowledgeEtl")
|
||||
public void knowledgeEtl(@RequestParam("files") MultipartFile[] files) {
|
||||
knowledgeEtlService.knowledgeEtl(files);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package som.supervision.knowsub.service;
|
||||
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public interface KnowledgeEtlService {
|
||||
|
||||
void knowledgeEtl(MultipartFile[] files);
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
package com.supervision.knowsub.config;
|
||||
|
||||
import org.elasticsearch.client.RestClient;
|
||||
import org.springframework.ai.autoconfigure.vectorstore.elasticsearch.ElasticsearchVectorStoreProperties;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.vectorstore.ElasticsearchVectorStore;
|
||||
import org.springframework.ai.vectorstore.ElasticsearchVectorStoreOptions;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(EmbeddingProperties.class)
|
||||
public class ElasticsearchVectorStoreConfig {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "embedding", name = "url")
|
||||
public EmbeddingModel embeddingModel(EmbeddingProperties embeddingProperties) {
|
||||
Assert.notNull(embeddingProperties.getUrl(), "配置文件embedding:url未找到");
|
||||
return new VectorEmbeddingModel(embeddingProperties.getUrl());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "embedding", name = "url")
|
||||
public ElasticsearchVectorStore vectorStore(ElasticsearchVectorStoreProperties properties,EmbeddingModel embeddingModel, RestClient restClient) {
|
||||
ElasticsearchVectorStoreOptions options = new ElasticsearchVectorStoreOptions();
|
||||
options.setIndexName(properties.getIndexName());
|
||||
options.setDimensions(1024);
|
||||
return new ElasticsearchVectorStore(options, restClient, embeddingModel, true);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
package com.supervision.knowsub.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "embedding")
|
||||
public class EmbeddingProperties {
|
||||
|
||||
private String url;
|
||||
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
package com.supervision.knowsub.config;
|
||||
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.*;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Slf4j
|
||||
public class VectorEmbeddingModel implements EmbeddingModel {
|
||||
|
||||
private final String embeddingUrl;
|
||||
|
||||
public VectorEmbeddingModel(String embeddingUrl) {
|
||||
this.embeddingUrl = embeddingUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Double> embed(Document document) {
|
||||
List<List<Double>> list = this.call(new EmbeddingRequest(List.of(document.getContent()), EmbeddingOptions.EMPTY))
|
||||
.getResults()
|
||||
.stream()
|
||||
.map(Embedding::getOutput)
|
||||
.toList();
|
||||
return list.iterator().next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public EmbeddingResponse call(EmbeddingRequest request) {
|
||||
Assert.notEmpty(request.getInstructions(), "At least one text is required!");
|
||||
List<List<Double>> embeddingList = new ArrayList<>();
|
||||
|
||||
for (String inputContent : request.getInstructions()) {
|
||||
// 这里需要吧inputContent转化为向量数据
|
||||
String post = HttpUtil.post(embeddingUrl, JSONUtil.toJsonStr(Map.of("text", inputContent)));
|
||||
EmbeddingData bean = JSONUtil.toBean(post, EmbeddingData.class);
|
||||
embeddingList.add(bean.embeddings);
|
||||
}
|
||||
var indexCounter = new AtomicInteger(0);
|
||||
List<Embedding> embeddings = embeddingList.stream()
|
||||
.map(e -> new Embedding(e, indexCounter.getAndIncrement()))
|
||||
.toList();
|
||||
return new EmbeddingResponse(embeddings);
|
||||
}
|
||||
|
||||
@Data
|
||||
private static class EmbeddingData {
|
||||
private List<Double> embeddings;
|
||||
}
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
package com.supervision.knowsub.controller;
|
||||
|
||||
import com.supervision.knowsub.dto.HtmlContext;
|
||||
import com.supervision.knowsub.etl.EtlProcessor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("etl")
|
||||
public class EtlController {
|
||||
|
||||
@Autowired
|
||||
private EtlProcessor etlProcessor;
|
||||
|
||||
@PostMapping("testLoadFile")
|
||||
public void testLoadFile(@RequestParam(name = "file") MultipartFile file) throws IOException {
|
||||
etlProcessor.loadFile(file.getInputStream());
|
||||
}
|
||||
|
||||
@PostMapping("testLoadHtml")
|
||||
public void testLoadHtml(@RequestBody HtmlContext htmlContext){
|
||||
etlProcessor.loadHtml(htmlContext);
|
||||
}
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
package com.supervision.knowsub.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class HtmlContext {
|
||||
|
||||
private String htmlContext;
|
||||
}
|
||||
@ -1,106 +1,94 @@
|
||||
package com.supervision.knowsub.util;
|
||||
|
||||
import cn.hutool.core.thread.ThreadUtil;
|
||||
import cn.hutool.json.JSONObject;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.ChatResponse;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.ollama.OllamaChatClient;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.*;
|
||||
|
||||
@Slf4j
|
||||
public class AiChatUtil {
|
||||
|
||||
private static final ExecutorService chatExecutor = ThreadUtil.newFixedExecutor(5, 5, "chat", new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
|
||||
private static final OllamaChatClient chatClient = SpringBeanUtil.getBean(OllamaChatClient.class);
|
||||
|
||||
/**
|
||||
* 单轮对话
|
||||
*
|
||||
* @param chat 对话的内容
|
||||
* @return jsonObject
|
||||
*/
|
||||
public static Optional<JSONObject> chat(String chat) {
|
||||
Prompt prompt = new Prompt(List.of(new UserMessage(chat)));
|
||||
Future<String> submit = chatExecutor.submit(new ChatTask(chatClient, prompt));
|
||||
try {
|
||||
return Optional.of(JSONUtil.parseObj(submit.get()));
|
||||
} catch (ExecutionException | InterruptedException e) {
|
||||
log.error("调用大模型生成失败");
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
/**
|
||||
* 支持多轮对话,自定义消息
|
||||
*
|
||||
* @param messageList 消息列表
|
||||
* @return jsonObject
|
||||
*/
|
||||
public static Optional<JSONObject> chat(List<Message> messageList) {
|
||||
Prompt prompt = new Prompt(messageList);
|
||||
Future<String> submit = chatExecutor.submit(new ChatTask(chatClient, prompt));
|
||||
try {
|
||||
return Optional.of(JSONUtil.parseObj(submit.get()));
|
||||
} catch (ExecutionException | InterruptedException e) {
|
||||
log.error("调用大模型生成失败");
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
/**
|
||||
* 支持序列化的方式
|
||||
*
|
||||
* @param messageList 消息列表
|
||||
* @param clazz 需要序列化的对象
|
||||
* @param <T> 需要序列化的对象的泛型
|
||||
* @return 对应对象类型, 不支持列表类型
|
||||
*/
|
||||
public static <T> Optional<T> chat(List<Message> messageList, Class<T> clazz) {
|
||||
Prompt prompt = new Prompt(messageList);
|
||||
Future<String> submit = chatExecutor.submit(new ChatTask(chatClient, prompt));
|
||||
try {
|
||||
String s = submit.get();
|
||||
return Optional.ofNullable(JSONUtil.toBean(s, clazz));
|
||||
} catch (ExecutionException | InterruptedException e) {
|
||||
log.error("调用大模型生成失败", e);
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
/**
|
||||
* 支持序列化的方式的对话
|
||||
*
|
||||
* @param chat 对话的消息
|
||||
* @param clazz 需要序列化的对象
|
||||
* @param <T> 需要序列化的对象的泛型
|
||||
* @return 对应对象类型, 不支持列表类型
|
||||
*/
|
||||
public static <T> Optional<T> chat(String chat, Class<T> clazz) {
|
||||
Prompt prompt = new Prompt(List.of(new UserMessage(chat)));
|
||||
Future<String> submit = chatExecutor.submit(new ChatTask(chatClient, prompt));
|
||||
try {
|
||||
String s = submit.get();
|
||||
return Optional.ofNullable(JSONUtil.toBean(s, clazz));
|
||||
} catch (ExecutionException | InterruptedException e) {
|
||||
log.error("调用大模型生成失败");
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
private record ChatTask(OllamaChatClient chatClient, Prompt prompt) implements Callable<String> {
|
||||
@Override
|
||||
public String call() {
|
||||
ChatResponse call = chatClient.call(prompt);
|
||||
return call.getResult().getOutput().getContent();
|
||||
}
|
||||
}
|
||||
// private static final ExecutorService chatExecutor = ThreadUtil.newFixedExecutor(5, 5, "chat", new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
//
|
||||
// private static final OllamaChatClient chatClient = SpringBeanUtil.getBean(OllamaChatClient.class);
|
||||
//
|
||||
// /**
|
||||
// * 单轮对话
|
||||
// *
|
||||
// * @param chat 对话的内容
|
||||
// * @return jsonObject
|
||||
// */
|
||||
// public static Optional<JSONObject> chat(String chat) {
|
||||
// Prompt prompt = new Prompt(List.of(new UserMessage(chat)));
|
||||
// Future<String> submit = chatExecutor.submit(new ChatTask(chatClient, prompt));
|
||||
// try {
|
||||
// return Optional.of(JSONUtil.parseObj(submit.get()));
|
||||
// } catch (ExecutionException | InterruptedException e) {
|
||||
// log.error("调用大模型生成失败");
|
||||
// }
|
||||
// return Optional.empty();
|
||||
// }
|
||||
//
|
||||
// /**
|
||||
// * 支持多轮对话,自定义消息
|
||||
// *
|
||||
// * @param messageList 消息列表
|
||||
// * @return jsonObject
|
||||
// */
|
||||
// public static Optional<JSONObject> chat(List<Message> messageList) {
|
||||
// Prompt prompt = new Prompt(messageList);
|
||||
// Future<String> submit = chatExecutor.submit(new ChatTask(chatClient, prompt));
|
||||
// try {
|
||||
// return Optional.of(JSONUtil.parseObj(submit.get()));
|
||||
// } catch (ExecutionException | InterruptedException e) {
|
||||
// log.error("调用大模型生成失败");
|
||||
// }
|
||||
// return Optional.empty();
|
||||
// }
|
||||
//
|
||||
// /**
|
||||
// * 支持序列化的方式
|
||||
// *
|
||||
// * @param messageList 消息列表
|
||||
// * @param clazz 需要序列化的对象
|
||||
// * @param <T> 需要序列化的对象的泛型
|
||||
// * @return 对应对象类型, 不支持列表类型
|
||||
// */
|
||||
// public static <T> Optional<T> chat(List<Message> messageList, Class<T> clazz) {
|
||||
// Prompt prompt = new Prompt(messageList);
|
||||
// Future<String> submit = chatExecutor.submit(new ChatTask(chatClient, prompt));
|
||||
// try {
|
||||
// String s = submit.get();
|
||||
// return Optional.ofNullable(JSONUtil.toBean(s, clazz));
|
||||
// } catch (ExecutionException | InterruptedException e) {
|
||||
// log.error("调用大模型生成失败", e);
|
||||
// }
|
||||
// return Optional.empty();
|
||||
// }
|
||||
//
|
||||
// /**
|
||||
// * 支持序列化的方式的对话
|
||||
// *
|
||||
// * @param chat 对话的消息
|
||||
// * @param clazz 需要序列化的对象
|
||||
// * @param <T> 需要序列化的对象的泛型
|
||||
// * @return 对应对象类型, 不支持列表类型
|
||||
// */
|
||||
// public static <T> Optional<T> chat(String chat, Class<T> clazz) {
|
||||
// Prompt prompt = new Prompt(List.of(new UserMessage(chat)));
|
||||
// Future<String> submit = chatExecutor.submit(new ChatTask(chatClient, prompt));
|
||||
// try {
|
||||
// String s = submit.get();
|
||||
// return Optional.ofNullable(JSONUtil.toBean(s, clazz));
|
||||
// } catch (ExecutionException | InterruptedException e) {
|
||||
// log.error("调用大模型生成失败");
|
||||
// }
|
||||
// return Optional.empty();
|
||||
// }
|
||||
//
|
||||
// private record ChatTask(OllamaChatClient chatClient, Prompt prompt) implements Callable<String> {
|
||||
// @Override
|
||||
// public String call() {
|
||||
// ChatResponse call = chatClient.call(prompt);
|
||||
// return call.getResult().getOutput().getContent();
|
||||
// }
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue