RAG代码提交
liu
8 months ago
parent
4fb5164193
commit
616678229f
@ -1,16 +0,0 @@
|
||||
package som.supervision.knowsub.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "vector.redis")
|
||||
public class RedisVectorProperties {
|
||||
|
||||
private String uri;
|
||||
|
||||
private String indexName;
|
||||
|
||||
private String prefix;
|
||||
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
package som.supervision.knowsub.config;
|
||||
|
||||
import org.springframework.ai.vectorstore.RedisVectorStore;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties({RedisVectorProperties.class, EmbeddingProperties.class})
|
||||
public class RedisVectorStoreConfig {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "embedding", name = "url")
|
||||
public VectorEmbeddingClient vectorEmbeddingClient(EmbeddingProperties embeddingProperties) {
|
||||
Assert.notNull(embeddingProperties.getUrl(), "配置文件embedding:url未找到");
|
||||
return new VectorEmbeddingClient(embeddingProperties.getUrl());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "vector.redis", name = "uri")
|
||||
public RedisVectorStore redisVectorStore(VectorEmbeddingClient vectorEmbeddingClient, RedisVectorProperties redisVectorProperties) {
|
||||
Assert.notNull(redisVectorProperties.getUri(), "配置文件vector.redis.uri未找到");
|
||||
RedisVectorStore.RedisVectorStoreConfig config = RedisVectorStore.RedisVectorStoreConfig.builder()
|
||||
.withURI(redisVectorProperties.getUri())
|
||||
.withPrefix(redisVectorProperties.getPrefix())
|
||||
.withIndexName(redisVectorProperties.getIndexName())
|
||||
// 定义搜索过滤器使用的元数据字段(!!!!!!!!千万重要,数据类型一定要用字符串,否则会导致查询不到!!!!!!!!)
|
||||
.withMetadataFields(
|
||||
RedisVectorStore.MetadataField.tag("fileName"))
|
||||
.build();
|
||||
return new RedisVectorStore(config, vectorEmbeddingClient);
|
||||
}
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
package som.supervision.knowsub.config;
|
||||
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.*;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Slf4j
|
||||
public class VectorEmbeddingClient extends AbstractEmbeddingClient {
|
||||
|
||||
private final String embeddingUrl;
|
||||
|
||||
public VectorEmbeddingClient(String embeddingUrl) {
|
||||
this.embeddingUrl = embeddingUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Double> embed(Document document) {
|
||||
List<List<Double>> list = this.call(new EmbeddingRequest(List.of(document.getContent()), EmbeddingOptions.EMPTY))
|
||||
.getResults()
|
||||
.stream()
|
||||
.map(Embedding::getOutput)
|
||||
.toList();
|
||||
return list.iterator().next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public EmbeddingResponse call(EmbeddingRequest request) {
|
||||
Assert.notEmpty(request.getInstructions(), "At least one text is required!");
|
||||
List<List<Double>> embeddingList = new ArrayList<>();
|
||||
|
||||
for (String inputContent : request.getInstructions()) {
|
||||
// 这里需要吧inputContent转化为向量数据
|
||||
String post = HttpUtil.post(embeddingUrl, JSONUtil.toJsonStr(Map.of("text", inputContent)));
|
||||
EmbeddingData bean = JSONUtil.toBean(post, EmbeddingData.class);
|
||||
embeddingList.add(bean.embeddings);
|
||||
}
|
||||
var indexCounter = new AtomicInteger(0);
|
||||
List<Embedding> embeddings = embeddingList.stream()
|
||||
.map(e -> new Embedding(e, indexCounter.getAndIncrement()))
|
||||
.toList();
|
||||
return new EmbeddingResponse(embeddings);
|
||||
}
|
||||
|
||||
@Data
|
||||
private static class EmbeddingData {
|
||||
private List<Double> embeddings;
|
||||
}
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
package com.supervision.knowsub.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "vector.redis")
|
||||
public class RedisVectorProperties {
|
||||
|
||||
private String uri;
|
||||
|
||||
private String indexName;
|
||||
|
||||
private String prefix;
|
||||
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
package com.supervision.knowsub.config;
|
||||
|
||||
import org.springframework.ai.vectorstore.RedisVectorStore;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties({RedisVectorProperties.class, EmbeddingProperties.class})
|
||||
public class RedisVectorStoreConfig {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "embedding", name = "url")
|
||||
public VectorEmbeddingClient vectorEmbeddingClient(EmbeddingProperties embeddingProperties) {
|
||||
Assert.notNull(embeddingProperties.getUrl(), "配置文件embedding:url未找到");
|
||||
return new VectorEmbeddingClient(embeddingProperties.getUrl());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "vector.redis", name = "uri")
|
||||
public RedisVectorStore redisVectorStore(VectorEmbeddingClient vectorEmbeddingClient, RedisVectorProperties redisVectorProperties) {
|
||||
Assert.notNull(redisVectorProperties.getUri(), "配置文件vector.redis.uri未找到");
|
||||
RedisVectorStore.RedisVectorStoreConfig config = RedisVectorStore.RedisVectorStoreConfig.builder()
|
||||
.withURI(redisVectorProperties.getUri())
|
||||
.withPrefix(redisVectorProperties.getPrefix())
|
||||
.withIndexName(redisVectorProperties.getIndexName())
|
||||
// 定义搜索过滤器使用的元数据字段(!!!!!!!!千万重要,数据类型一定要用字符串,否则会导致查询不到!!!!!!!!)
|
||||
.withMetadataFields(
|
||||
RedisVectorStore.MetadataField.tag("fileName"))
|
||||
.build();
|
||||
return new RedisVectorStore(config, vectorEmbeddingClient);
|
||||
}
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
package com.supervision.knowsub.config;
|
||||
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.*;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@Slf4j
|
||||
public class VectorEmbeddingClient extends AbstractEmbeddingClient {
|
||||
|
||||
private final String embeddingUrl;
|
||||
|
||||
public VectorEmbeddingClient(String embeddingUrl) {
|
||||
this.embeddingUrl = embeddingUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Double> embed(Document document) {
|
||||
List<List<Double>> list = this.call(new EmbeddingRequest(List.of(document.getContent()), EmbeddingOptions.EMPTY))
|
||||
.getResults()
|
||||
.stream()
|
||||
.map(Embedding::getOutput)
|
||||
.toList();
|
||||
return list.iterator().next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public EmbeddingResponse call(EmbeddingRequest request) {
|
||||
Assert.notEmpty(request.getInstructions(), "At least one text is required!");
|
||||
List<List<Double>> embeddingList = new ArrayList<>();
|
||||
|
||||
for (String inputContent : request.getInstructions()) {
|
||||
// 这里需要吧inputContent转化为向量数据
|
||||
String post = HttpUtil.post(embeddingUrl, JSONUtil.toJsonStr(Map.of("text", inputContent)));
|
||||
EmbeddingData bean = JSONUtil.toBean(post, EmbeddingData.class);
|
||||
embeddingList.add(bean.embeddings);
|
||||
}
|
||||
var indexCounter = new AtomicInteger(0);
|
||||
List<Embedding> embeddings = embeddingList.stream()
|
||||
.map(e -> new Embedding(e, indexCounter.getAndIncrement()))
|
||||
.toList();
|
||||
return new EmbeddingResponse(embeddings);
|
||||
}
|
||||
|
||||
@Data
|
||||
private static class EmbeddingData {
|
||||
private List<Double> embeddings;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue