新增模块
liu
10 months ago
parent
6056be868b
commit
6eb2f5f7e1
@ -0,0 +1,110 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>com.supervision</groupId>
|
||||||
|
<artifactId>know_sub</artifactId>
|
||||||
|
<version>1.0.0</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<artifactId>know_sub_rag</artifactId>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<maven.compiler.source>17</maven.compiler.source>
|
||||||
|
<maven.compiler.target>17</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
|
||||||
|
<!-- 引入ollama的依赖.版本号来自于 dependencyManagement中 spring-ai-bom中的版本号.-->
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.springboot.ai</groupId>
|
||||||
|
<artifactId>spring-ai-ollama-spring-boot-starter</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.springboot.ai</groupId>
|
||||||
|
<artifactId>spring-ai-elasticsearch-store</artifactId>
|
||||||
|
<version>1.0.3</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-configuration-processor</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-starter-test</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.supervision</groupId>
|
||||||
|
<artifactId>know_sub_common</artifactId>
|
||||||
|
<version>1.0.0</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.supervision</groupId>
|
||||||
|
<artifactId>know_sub_model</artifactId>
|
||||||
|
<version>1.0.0</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.projectlombok</groupId>
|
||||||
|
<artifactId>lombok</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||||
|
<version>3.3.0</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<goals>
|
||||||
|
<goal>repackage</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<excludes>
|
||||||
|
<exclude>
|
||||||
|
<groupId>org.projectlombok</groupId>
|
||||||
|
<artifactId>lombok</artifactId>
|
||||||
|
</exclude>
|
||||||
|
</excludes>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<version>3.11.0</version>
|
||||||
|
<configuration>
|
||||||
|
<parameters>true</parameters>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
</project>
|
||||||
@ -0,0 +1,27 @@
|
|||||||
|
package com.supervision.knowsub.config;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.RestClient;
|
||||||
|
import org.springframework.ai.vectorstore.ElasticsearchVectorStore;
|
||||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||||
|
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
import org.springframework.util.Assert;
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
@EnableConfigurationProperties(EmbeddingProperties.class)
|
||||||
|
public class ElasticsearchVectorStoreConfig {
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
// @ConditionalOnProperty(prefix = "embedding", name = "url")
|
||||||
|
public VectorEmbeddingClient vectorEmbeddingClient(EmbeddingProperties embeddingProperties) {
|
||||||
|
Assert.notNull(embeddingProperties.getUrl(), "配置文件embedding:url未找到");
|
||||||
|
return new VectorEmbeddingClient(embeddingProperties.getUrl());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public ElasticsearchVectorStore vectorStore(VectorEmbeddingClient embeddingModel, RestClient restClient) {
|
||||||
|
return new ElasticsearchVectorStore(restClient, embeddingModel);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
package com.supervision.knowsub.config;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@ConfigurationProperties(prefix = "embedding")
|
||||||
|
public class EmbeddingProperties {
|
||||||
|
|
||||||
|
private String url;
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,57 @@
|
|||||||
|
package com.supervision.knowsub.config;
|
||||||
|
|
||||||
|
import cn.hutool.http.HttpUtil;
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.ai.document.Document;
|
||||||
|
import org.springframework.ai.embedding.*;
|
||||||
|
import org.springframework.util.Assert;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class VectorEmbeddingClient extends AbstractEmbeddingClient {
|
||||||
|
|
||||||
|
private final String embeddingUrl;
|
||||||
|
|
||||||
|
public VectorEmbeddingClient(String embeddingUrl) {
|
||||||
|
this.embeddingUrl = embeddingUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Double> embed(Document document) {
|
||||||
|
List<List<Double>> list = this.call(new EmbeddingRequest(List.of(document.getContent()), EmbeddingOptions.EMPTY))
|
||||||
|
.getResults()
|
||||||
|
.stream()
|
||||||
|
.map(Embedding::getOutput)
|
||||||
|
.toList();
|
||||||
|
return list.iterator().next();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EmbeddingResponse call(EmbeddingRequest request) {
|
||||||
|
Assert.notEmpty(request.getInstructions(), "At least one text is required!");
|
||||||
|
List<List<Double>> embeddingList = new ArrayList<>();
|
||||||
|
|
||||||
|
for (String inputContent : request.getInstructions()) {
|
||||||
|
// 这里需要吧inputContent转化为向量数据
|
||||||
|
String post = HttpUtil.post(embeddingUrl, JSONUtil.toJsonStr(Map.of("text", inputContent)));
|
||||||
|
EmbeddingData bean = JSONUtil.toBean(post, EmbeddingData.class);
|
||||||
|
embeddingList.add(bean.embeddings);
|
||||||
|
}
|
||||||
|
var indexCounter = new AtomicInteger(0);
|
||||||
|
List<Embedding> embeddings = embeddingList.stream()
|
||||||
|
.map(e -> new Embedding(e, indexCounter.getAndIncrement()))
|
||||||
|
.toList();
|
||||||
|
return new EmbeddingResponse(embeddings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
private static class EmbeddingData {
|
||||||
|
private List<Double> embeddings;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
spring:
|
||||||
|
elasticsearch:
|
||||||
|
uris: http://192.168.10.137:9200
|
||||||
|
|
||||||
|
embedding:
|
||||||
|
url: http://192.168.10.137:8711/embeddings/
|
||||||
Loading…
Reference in New Issue