RAG代码提交
liu
8 months ago
parent
665ee4af13
commit
4fb5164193
@ -0,0 +1,16 @@
|
||||
package som.supervision.knowsub.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "vector.redis")
|
||||
public class RedisVectorProperties {
|
||||
|
||||
private String uri;
|
||||
|
||||
private String indexName;
|
||||
|
||||
private String prefix;
|
||||
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package som.supervision.knowsub.config;
|
||||
|
||||
import org.springframework.ai.vectorstore.RedisVectorStore;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties({RedisVectorProperties.class, EmbeddingProperties.class})
|
||||
public class RedisVectorStoreConfig {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "embedding", name = "url")
|
||||
public VectorEmbeddingClient vectorEmbeddingClient(EmbeddingProperties embeddingProperties) {
|
||||
Assert.notNull(embeddingProperties.getUrl(), "配置文件embedding:url未找到");
|
||||
return new VectorEmbeddingClient(embeddingProperties.getUrl());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "vector.redis", name = "uri")
|
||||
public RedisVectorStore redisVectorStore(VectorEmbeddingClient vectorEmbeddingClient, RedisVectorProperties redisVectorProperties) {
|
||||
Assert.notNull(redisVectorProperties.getUri(), "配置文件vector.redis.uri未找到");
|
||||
RedisVectorStore.RedisVectorStoreConfig config = RedisVectorStore.RedisVectorStoreConfig.builder()
|
||||
.withURI(redisVectorProperties.getUri())
|
||||
.withPrefix(redisVectorProperties.getPrefix())
|
||||
.withIndexName(redisVectorProperties.getIndexName())
|
||||
// 定义搜索过滤器使用的元数据字段(!!!!!!!!千万重要,数据类型一定要用字符串,否则会导致查询不到!!!!!!!!)
|
||||
.withMetadataFields(
|
||||
RedisVectorStore.MetadataField.tag("fileName"))
|
||||
.build();
|
||||
return new RedisVectorStore(config, vectorEmbeddingClient);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
package som.supervision.knowsub.config;
|
||||
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.*;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Slf4j
|
||||
public class VectorEmbeddingClient extends AbstractEmbeddingClient {
|
||||
|
||||
private final String embeddingUrl;
|
||||
|
||||
public VectorEmbeddingClient(String embeddingUrl) {
|
||||
this.embeddingUrl = embeddingUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Double> embed(Document document) {
|
||||
List<List<Double>> list = this.call(new EmbeddingRequest(List.of(document.getContent()), EmbeddingOptions.EMPTY))
|
||||
.getResults()
|
||||
.stream()
|
||||
.map(Embedding::getOutput)
|
||||
.toList();
|
||||
return list.iterator().next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public EmbeddingResponse call(EmbeddingRequest request) {
|
||||
Assert.notEmpty(request.getInstructions(), "At least one text is required!");
|
||||
List<List<Double>> embeddingList = new ArrayList<>();
|
||||
|
||||
for (String inputContent : request.getInstructions()) {
|
||||
// 这里需要吧inputContent转化为向量数据
|
||||
String post = HttpUtil.post(embeddingUrl, JSONUtil.toJsonStr(Map.of("text", inputContent)));
|
||||
EmbeddingData bean = JSONUtil.toBean(post, EmbeddingData.class);
|
||||
embeddingList.add(bean.embeddings);
|
||||
}
|
||||
var indexCounter = new AtomicInteger(0);
|
||||
List<Embedding> embeddings = embeddingList.stream()
|
||||
.map(e -> new Embedding(e, indexCounter.getAndIncrement()))
|
||||
.toList();
|
||||
return new EmbeddingResponse(embeddings);
|
||||
}
|
||||
|
||||
@Data
|
||||
private static class EmbeddingData {
|
||||
private List<Double> embeddings;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package com.supervision.knowsub.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "vector.redis")
|
||||
public class RedisVectorProperties {
|
||||
|
||||
private String uri;
|
||||
|
||||
private String indexName;
|
||||
|
||||
private String prefix;
|
||||
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package com.supervision.knowsub.config;
|
||||
|
||||
import org.springframework.ai.vectorstore.RedisVectorStore;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties({RedisVectorProperties.class, EmbeddingProperties.class})
|
||||
public class RedisVectorStoreConfig {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "embedding", name = "url")
|
||||
public VectorEmbeddingClient vectorEmbeddingClient(EmbeddingProperties embeddingProperties) {
|
||||
Assert.notNull(embeddingProperties.getUrl(), "配置文件embedding:url未找到");
|
||||
return new VectorEmbeddingClient(embeddingProperties.getUrl());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "vector.redis", name = "uri")
|
||||
public RedisVectorStore redisVectorStore(VectorEmbeddingClient vectorEmbeddingClient, RedisVectorProperties redisVectorProperties) {
|
||||
Assert.notNull(redisVectorProperties.getUri(), "配置文件vector.redis.uri未找到");
|
||||
RedisVectorStore.RedisVectorStoreConfig config = RedisVectorStore.RedisVectorStoreConfig.builder()
|
||||
.withURI(redisVectorProperties.getUri())
|
||||
.withPrefix(redisVectorProperties.getPrefix())
|
||||
.withIndexName(redisVectorProperties.getIndexName())
|
||||
// 定义搜索过滤器使用的元数据字段(!!!!!!!!千万重要,数据类型一定要用字符串,否则会导致查询不到!!!!!!!!)
|
||||
.withMetadataFields(
|
||||
RedisVectorStore.MetadataField.tag("fileName"))
|
||||
.build();
|
||||
return new RedisVectorStore(config, vectorEmbeddingClient);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
package com.supervision.knowsub.config;
|
||||
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.*;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Slf4j
|
||||
public class VectorEmbeddingClient extends AbstractEmbeddingClient {
|
||||
|
||||
private final String embeddingUrl;
|
||||
|
||||
public VectorEmbeddingClient(String embeddingUrl) {
|
||||
this.embeddingUrl = embeddingUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Double> embed(Document document) {
|
||||
List<List<Double>> list = this.call(new EmbeddingRequest(List.of(document.getContent()), EmbeddingOptions.EMPTY))
|
||||
.getResults()
|
||||
.stream()
|
||||
.map(Embedding::getOutput)
|
||||
.toList();
|
||||
return list.iterator().next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public EmbeddingResponse call(EmbeddingRequest request) {
|
||||
Assert.notEmpty(request.getInstructions(), "At least one text is required!");
|
||||
List<List<Double>> embeddingList = new ArrayList<>();
|
||||
|
||||
for (String inputContent : request.getInstructions()) {
|
||||
// 这里需要吧inputContent转化为向量数据
|
||||
String post = HttpUtil.post(embeddingUrl, JSONUtil.toJsonStr(Map.of("text", inputContent)));
|
||||
EmbeddingData bean = JSONUtil.toBean(post, EmbeddingData.class);
|
||||
embeddingList.add(bean.embeddings);
|
||||
}
|
||||
var indexCounter = new AtomicInteger(0);
|
||||
List<Embedding> embeddings = embeddingList.stream()
|
||||
.map(e -> new Embedding(e, indexCounter.getAndIncrement()))
|
||||
.toList();
|
||||
return new EmbeddingResponse(embeddings);
|
||||
}
|
||||
|
||||
@Data
|
||||
private static class EmbeddingData {
|
||||
private List<Double> embeddings;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,56 @@
|
||||
package com.supervision.knowsub.controller;
|
||||
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import cn.hutool.poi.excel.ExcelReader;
|
||||
import cn.hutool.poi.excel.ExcelUtil;
|
||||
import cn.hutool.poi.excel.ExcelWriter;
|
||||
import com.supervision.knowsub.service.RagService;
|
||||
import com.supervision.knowsub.vo.RagResVO;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("test")
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class TestController {
|
||||
|
||||
private final RagService ragService;
|
||||
|
||||
@GetMapping("esTest")
|
||||
public void esTest(){
|
||||
ExcelReader reader = ExcelUtil.getReader("/Users/flevance/Desktop/深圳人社POC/question.xlsx","Sheet2");
|
||||
List<Object> objects = reader.readColumn(3, 1);
|
||||
ExcelWriter writer = reader.getWriter();
|
||||
|
||||
for (int i = 0; i < objects.size(); i++) {
|
||||
RagResVO ask = ragService.esAsk(objects.get(i).toString());
|
||||
|
||||
writer.writeCellValue(5, i + 1, ask.getAnswer());
|
||||
writer.writeCellValue(6, i + 1, JSONUtil.toJsonStr(ask.getFileName()));
|
||||
log.info("第{}条数据写入成功,剩余{}条", i + 1, objects.size() - i - 1);
|
||||
}
|
||||
writer.flush();
|
||||
}
|
||||
|
||||
@GetMapping("redisTest")
|
||||
public void redisTest(){
|
||||
ExcelReader reader = ExcelUtil.getReader("/Users/flevance/Desktop/深圳人社POC/question.xlsx","Sheet2");
|
||||
List<Object> objects = reader.readColumn(3, 1);
|
||||
ExcelWriter writer = reader.getWriter();
|
||||
|
||||
for (int i = 0; i < objects.size(); i++) {
|
||||
RagResVO ask = ragService.redisAsk(objects.get(i).toString());
|
||||
|
||||
writer.writeCellValue(5, i + 1, ask.getAnswer());
|
||||
writer.writeCellValue(6, i + 1, JSONUtil.toJsonStr(ask.getFileName()));
|
||||
log.info("第{}条数据写入成功,剩余{}条", i + 1, objects.size() - i - 1);
|
||||
}
|
||||
writer.flush();
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,10 @@
|
||||
package com.supervision.knowsub.service;
|
||||
|
||||
import com.supervision.knowsub.vo.RagResVO;
|
||||
|
||||
public interface RagService {
|
||||
|
||||
String ask(String question);
|
||||
RagResVO esAsk(String question);
|
||||
|
||||
RagResVO redisAsk(String question);
|
||||
}
|
||||
|
||||
@ -0,0 +1,14 @@
|
||||
package com.supervision.knowsub.vo;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@Data
|
||||
public class RagResVO {
|
||||
|
||||
private String answer;
|
||||
|
||||
private Set<String> fileName;
|
||||
}
|
||||
Loading…
Reference in New Issue