新增模块

know_sub_etl/src/main/java/som/supervision/knowsub/config/ElasticsearchVectorStoreConfig.java

@@ -0,0 +1,33 @@
+package som.supervision.knowsub.config;
+
+import org.elasticsearch.client.RestClient;
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.vectorstore.ElasticsearchVectorStore;
+import org.springframework.ai.vectorstore.ElasticsearchVectorStoreOptions;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.util.Assert;
+
+@Configuration
+@EnableConfigurationProperties(EmbeddingProperties.class)
+public class ElasticsearchVectorStoreConfig {
+
+    @Bean
+    @ConditionalOnProperty(prefix = "embedding", name = "url")
+    public EmbeddingModel embeddingModel(EmbeddingProperties embeddingProperties) {
+        Assert.notNull(embeddingProperties.getUrl(), "配置文件embedding:url未找到");
+        return new VectorEmbeddingModel(embeddingProperties.getUrl());
+    }
+
+    @Bean
+    @ConditionalOnProperty(prefix = "embedding", name = "url")
+    public ElasticsearchVectorStore vectorStore(EmbeddingModel embeddingModel, RestClient restClient) {
+        ElasticsearchVectorStoreOptions options = new ElasticsearchVectorStoreOptions();
+        options.setIndexName("test_rag");
+        options.setDimensions(1024);
+        return new ElasticsearchVectorStore(options, restClient, embeddingModel, true);
+    }
+
+}

know_sub_etl/src/main/java/som/supervision/knowsub/config/EmbeddingProperties.java

@@ -0,0 +1,12 @@
+package som.supervision.knowsub.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+@Data
+@ConfigurationProperties(prefix = "embedding")
+public class EmbeddingProperties {
+
+    private String url;
+
+}

know_sub_etl/src/main/java/som/supervision/knowsub/config/VectorEmbeddingModel.java

@@ -0,0 +1,57 @@
+package som.supervision.knowsub.config;
+
+import cn.hutool.http.HttpUtil;
+import cn.hutool.json.JSONUtil;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.embedding.*;
+import org.springframework.util.Assert;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Slf4j
+public class VectorEmbeddingModel implements EmbeddingModel {
+
+    private final String embeddingUrl;
+
+    public VectorEmbeddingModel(String embeddingUrl) {
+        this.embeddingUrl = embeddingUrl;
+    }
+
+    @Override
+    public List<Double> embed(Document document) {
+        List<List<Double>> list = this.call(new EmbeddingRequest(List.of(document.getContent()), EmbeddingOptions.EMPTY))
+                .getResults()
+                .stream()
+                .map(Embedding::getOutput)
+                .toList();
+        return list.iterator().next();
+    }
+
+    @Override
+    public EmbeddingResponse call(EmbeddingRequest request) {
+        Assert.notEmpty(request.getInstructions(), "At least one text is required!");
+        List<List<Double>> embeddingList = new ArrayList<>();
+
+        for (String inputContent : request.getInstructions()) {
+            // 这里需要吧inputContent转化为向量数据
+            String post = HttpUtil.post(embeddingUrl, JSONUtil.toJsonStr(Map.of("text", inputContent)));
+            EmbeddingData bean = JSONUtil.toBean(post, EmbeddingData.class);
+            embeddingList.add(bean.embeddings);
+        }
+        var indexCounter = new AtomicInteger(0);
+        List<Embedding> embeddings = embeddingList.stream()
+                .map(e -> new Embedding(e, indexCounter.getAndIncrement()))
+                .toList();
+        return new EmbeddingResponse(embeddings);
+    }
+
+    @Data
+    private static class EmbeddingData {
+        private List<Double> embeddings;
+    }
+}

know_sub_etl/src/main/resources/application.yml

@@ -17,7 +17,7 @@ spring:
   ai:
     vectorstore:
       elasticsearch:
-        index-name: 11
+        index-name: know-sub-rag-store
 
   main:
     allow-bean-definition-overriding: true

know_sub_rag/src/main/resources/application.yml

@@ -17,7 +17,7 @@ spring:
   ai:
     vectorstore:
       elasticsearch:
-        index-name: 11
+        index-name: know-sub-rag-store
 
   main:
     allow-bean-definition-overriding: true