问答功能优化-初始化表

v_0.0.2
xueqingkun 1 day ago
parent fd4641cc48
commit 2ea04d7325

@ -1,6 +1,7 @@
package com.supervision.pdfqaserver.config;
import com.baomidou.mybatisplus.annotation.DbType;
import com.baomidou.mybatisplus.autoconfigure.ConfigurationCustomizer;
import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
import org.springframework.context.annotation.Bean;
@ -31,6 +32,16 @@ public class MybatisPlusConfig {
return interceptor;
}
@Bean
public ConfigurationCustomizer mybatisConfigurationCustomizer() {
return configuration -> {
configuration.getTypeHandlerRegistry()
.register(float[].class, new VectorTypeHandler());
};
}
private PaginationInnerInterceptor paginationInterceptor() {
PaginationInnerInterceptor paginationInterceptor = new PaginationInnerInterceptor();
paginationInterceptor.setOverflow(false);

@ -0,0 +1,56 @@
package com.supervision.pdfqaserver.config;
import org.apache.ibatis.type.BaseTypeHandler;
import org.apache.ibatis.type.JdbcType;
import org.postgresql.util.PGobject;
import java.sql.*;
public class VectorTypeHandler extends BaseTypeHandler<float[]> {
@Override
public void setNonNullParameter(PreparedStatement ps, int i,
float[] parameter, JdbcType jdbcType) throws SQLException {
// 将float数组转换为PostgreSQL vector格式字符串
PGobject vector = new PGobject();
vector.setType("vector");
vector.setValue(arrayToVectorString(parameter));
ps.setObject(i,vector); ;
}
@Override
public float[] getNullableResult(ResultSet rs, String columnName) throws SQLException {
return vectorStringToArray(rs.getString(columnName));
}
@Override
public float[] getNullableResult(ResultSet rs, int columnIndex) throws SQLException {
return vectorStringToArray(rs.getString(columnIndex));
}
@Override
public float[] getNullableResult(CallableStatement cs, int columnIndex) throws SQLException {
return vectorStringToArray(cs.getString(columnIndex));
}
private String arrayToVectorString(float[] array) {
StringBuilder sb = new StringBuilder("[");
for (int i = 0; i < array.length; i++) {
if (i > 0) sb.append(",");
sb.append(array[i]);
}
sb.append("]");
return sb.toString();
}
private float[] vectorStringToArray(String vector) {
if (vector == null || vector.isEmpty()) return null;
String cleaned = vector.replace("[", "").replace("]", "");
String[] parts = cleaned.split(",");
float[] result = new float[parts.length];
for (int i = 0; i < parts.length; i++) {
result[i] = Float.parseFloat(parts[i].trim());
}
return result;
}
}

@ -0,0 +1,46 @@
package com.supervision.pdfqaserver.domain;
import com.baomidou.mybatisplus.annotation.*;
import java.io.Serializable;
import java.time.LocalDateTime;
import lombok.Data;
/**
*
* @TableName question_category
*/
@TableName(value ="question_category")
@Data
public class QuestionCategory implements Serializable {
/**
*
*/
@TableId
private String id;
/**
*
*/
private String categoryName;
/**
* id
*/
private String parentId;
/**
*
*/
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
/**
*
*/
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
@TableField(exist = false)
private static final long serialVersionUID = 1L;
}

@ -0,0 +1,45 @@
package com.supervision.pdfqaserver.domain;
import com.baomidou.mybatisplus.annotation.*;
import java.io.Serializable;
import java.time.LocalDateTime;
import lombok.Data;
/**
*
* @TableName question_category_intention
*/
@TableName(value ="question_category_intention")
@Data
public class QuestionCategoryIntention implements Serializable {
/**
*
*/
@TableId
private String id;
/**
* id
*/
private String categoryId;
/**
* id
*/
private String intentId;
/**
*
*/
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
/**
*
*/
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
@TableField(exist = false)
private static final long serialVersionUID = 1L;
}

@ -0,0 +1,46 @@
package com.supervision.pdfqaserver.domain;
import com.baomidou.mybatisplus.annotation.*;
import java.io.Serializable;
import java.time.LocalDateTime;
import lombok.Data;
/**
*
* @TableName question_handler_mapping
*/
@TableName(value ="question_handler_mapping")
@Data
public class QuestionHandlerMapping implements Serializable {
/**
*
*/
@TableId
private String id;
/**
* id
*/
private String questionCategoryId;
/**
*
*/
private String handler;
/**
*
*/
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
/**
*
*/
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
@TableField(exist = false)
private static final long serialVersionUID = 1L;
}

@ -0,0 +1,54 @@
package com.supervision.pdfqaserver.domain;
import com.baomidou.mybatisplus.annotation.*;
import java.io.Serializable;
import java.time.LocalDateTime;
import com.supervision.pdfqaserver.config.VectorTypeHandler;
import lombok.Data;
/**
*
* @TableName text_vector
*/
@TableName(value ="text_vector", autoResultMap = true)
@Data
public class TextVector implements Serializable {
/**
*
*/
@TableId
private String id;
/**
*
*/
private String content;
/**
*
*/
@TableField(typeHandler = VectorTypeHandler.class)
private float[] embedding;
/**
* id
*/
private String categoryId;
/**
*
*/
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
/**
*
*/
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
@TableField(exist = false)
private static final long serialVersionUID = 1L;
}

@ -0,0 +1,31 @@
package com.supervision.pdfqaserver.dto;
import lombok.Data;
@Data
public class TextVectorDTO {
private String id;
private double similarityScore;
/**
*
*/
private String content;
/**
* id
*/
private String categoryId;
/**
*
*/
private float[] embedding;
}

@ -0,0 +1,18 @@
package com.supervision.pdfqaserver.mapper;
import com.supervision.pdfqaserver.domain.QuestionCategoryIntention;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* @author Administrator
* @description question_category_intention()Mapper
* @createDate 2025-06-13 11:29:01
* @Entity com.supervision.pdfqaserver.domain.QuestionCategoryIntention
*/
public interface QuestionCategoryIntentionMapper extends BaseMapper<QuestionCategoryIntention> {
}

@ -0,0 +1,18 @@
package com.supervision.pdfqaserver.mapper;
import com.supervision.pdfqaserver.domain.QuestionCategory;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* @author Administrator
* @description question_category()Mapper
* @createDate 2025-06-13 11:29:01
* @Entity com.supervision.pdfqaserver.domain.QuestionCategory
*/
public interface QuestionCategoryMapper extends BaseMapper<QuestionCategory> {
}

@ -0,0 +1,18 @@
package com.supervision.pdfqaserver.mapper;
import com.supervision.pdfqaserver.domain.QuestionHandlerMapping;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* @author Administrator
* @description question_handler_mapping()Mapper
* @createDate 2025-06-13 11:29:01
* @Entity com.supervision.pdfqaserver.domain.QuestionHandlerMapping
*/
public interface QuestionHandlerMappingMapper extends BaseMapper<QuestionHandlerMapping> {
}

@ -0,0 +1,23 @@
package com.supervision.pdfqaserver.mapper;
import com.supervision.pdfqaserver.domain.TextVector;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.supervision.pdfqaserver.dto.TextVectorDTO;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/**
* @author Administrator
* @description text_vector()Mapper
* @createDate 2025-06-11 16:40:57
* @Entity com.supervision.pdfqaserver.domain.TextVector
*/
public interface TextVectorMapper extends BaseMapper<TextVector> {
List<TextVectorDTO> findSimilarByCosine(@Param("embedding")float[] embedding, @Param("threshold") double threshold, @Param("limit")int limit);
}

@ -2,6 +2,7 @@ package com.supervision.pdfqaserver.service;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.embedding.Embedding;
import reactor.core.publisher.Flux;
/**
@ -14,5 +15,5 @@ public interface AiCallService {
Flux<ChatResponse> stream(Prompt prompt);
abstract void embedding(String text);
Embedding embedding(String text);
}

@ -4,6 +4,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.embedding.Embedding;
import org.springframework.ai.openai.OpenAiChatModel;
import reactor.core.publisher.Flux;
import org.springframework.stereotype.Service;
@ -28,7 +29,8 @@ public class DeepSeekApiImpl implements AiCallService {
}
@Override
public void embedding(String text) {
public Embedding embedding(String text) {
return null;
}
}

@ -0,0 +1,13 @@
package com.supervision.pdfqaserver.service;
import com.supervision.pdfqaserver.domain.QuestionCategoryIntention;
import com.baomidou.mybatisplus.extension.service.IService;
/**
* @author Administrator
* @description question_category_intention()Service
* @createDate 2025-06-13 11:29:01
*/
public interface QuestionCategoryIntentionService extends IService<QuestionCategoryIntention> {
}

@ -0,0 +1,13 @@
package com.supervision.pdfqaserver.service;
import com.supervision.pdfqaserver.domain.QuestionCategory;
import com.baomidou.mybatisplus.extension.service.IService;
/**
* @author Administrator
* @description question_category()Service
* @createDate 2025-06-13 11:29:01
*/
public interface QuestionCategoryService extends IService<QuestionCategory> {
}

@ -0,0 +1,13 @@
package com.supervision.pdfqaserver.service;
import com.supervision.pdfqaserver.domain.QuestionHandlerMapping;
import com.baomidou.mybatisplus.extension.service.IService;
/**
* @author Administrator
* @description question_handler_mapping()Service
* @createDate 2025-06-13 11:29:01
*/
public interface QuestionHandlerMappingService extends IService<QuestionHandlerMapping> {
}

@ -0,0 +1,18 @@
package com.supervision.pdfqaserver.service;
import com.supervision.pdfqaserver.domain.TextVector;
import com.baomidou.mybatisplus.extension.service.IService;
import com.supervision.pdfqaserver.dto.TextVectorDTO;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/**
* @author Administrator
* @description text_vector()Service
* @createDate 2025-06-11 16:40:57
*/
public interface TextVectorService extends IService<TextVector> {
List<TextVectorDTO> findSimilarByCosine(float[] embedding, double threshold , int limit);
}

@ -32,12 +32,9 @@ public class OllamaCallServiceImpl implements AiCallService {
return ollamaChatModel.stream(prompt);
}
public void embedding(String text) {
public Embedding embedding(String text) {
EmbeddingResponse embeddingResponse = embeddingModel.call(
new EmbeddingRequest(List.of("Hello World", "World is big and salvation is near"),
OllamaOptions.builder().model("quentinz/bge-large-zh-v1.5:latest").build()));
Embedding result = embeddingResponse.getResult();
System.out.println(result);
EmbeddingResponse embeddingResponse = embeddingModel.call(new EmbeddingRequest(List.of(text),null));
return embeddingResponse.getResult();
}
}

@ -0,0 +1,22 @@
package com.supervision.pdfqaserver.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.pdfqaserver.domain.QuestionCategoryIntention;
import com.supervision.pdfqaserver.service.QuestionCategoryIntentionService;
import com.supervision.pdfqaserver.mapper.QuestionCategoryIntentionMapper;
import org.springframework.stereotype.Service;
/**
* @author Administrator
* @description question_category_intention()Service
* @createDate 2025-06-13 11:29:01
*/
@Service
public class QuestionCategoryIntentionServiceImpl extends ServiceImpl<QuestionCategoryIntentionMapper, QuestionCategoryIntention>
implements QuestionCategoryIntentionService{
}

@ -0,0 +1,22 @@
package com.supervision.pdfqaserver.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.pdfqaserver.domain.QuestionCategory;
import com.supervision.pdfqaserver.service.QuestionCategoryService;
import com.supervision.pdfqaserver.mapper.QuestionCategoryMapper;
import org.springframework.stereotype.Service;
/**
* @author Administrator
* @description question_category()Service
* @createDate 2025-06-13 11:29:01
*/
@Service
public class QuestionCategoryServiceImpl extends ServiceImpl<QuestionCategoryMapper, QuestionCategory>
implements QuestionCategoryService{
}

@ -0,0 +1,22 @@
package com.supervision.pdfqaserver.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.pdfqaserver.domain.QuestionHandlerMapping;
import com.supervision.pdfqaserver.service.QuestionHandlerMappingService;
import com.supervision.pdfqaserver.mapper.QuestionHandlerMappingMapper;
import org.springframework.stereotype.Service;
/**
* @author Administrator
* @description question_handler_mapping()Service
* @createDate 2025-06-13 11:29:01
*/
@Service
public class QuestionHandlerMappingServiceImpl extends ServiceImpl<QuestionHandlerMappingMapper, QuestionHandlerMapping>
implements QuestionHandlerMappingService{
}

@ -0,0 +1,29 @@
package com.supervision.pdfqaserver.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.pdfqaserver.domain.TextVector;
import com.supervision.pdfqaserver.dto.TextVectorDTO;
import com.supervision.pdfqaserver.service.TextVectorService;
import com.supervision.pdfqaserver.mapper.TextVectorMapper;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* @author Administrator
* @description text_vector()Service
* @createDate 2025-06-11 16:40:57
*/
@Service
public class TextVectorServiceImpl extends ServiceImpl<TextVectorMapper, TextVector>
implements TextVectorService{
@Override
public List<TextVectorDTO> findSimilarByCosine(float[] embedding, double threshold , int limit) {
return super.getBaseMapper().findSimilarByCosine(embedding, threshold,limit);
}
}

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.supervision.pdfqaserver.mapper.QuestionCategoryIntentionMapper">
<resultMap id="BaseResultMap" type="com.supervision.pdfqaserver.domain.QuestionCategoryIntention">
<id property="id" column="id" jdbcType="VARCHAR"/>
<result property="categoryId" column="category_id" jdbcType="VARCHAR"/>
<result property="intentId" column="intent_id" jdbcType="VARCHAR"/>
<result property="createTime" column="create_time" jdbcType="TIMESTAMP"/>
<result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
</resultMap>
<sql id="Base_Column_List">
id,category_id,intent_id,
create_time,update_time
</sql>
</mapper>

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.supervision.pdfqaserver.mapper.QuestionCategoryMapper">
<resultMap id="BaseResultMap" type="com.supervision.pdfqaserver.domain.QuestionCategory">
<id property="id" column="id" jdbcType="VARCHAR"/>
<result property="categoryName" column="category_name" jdbcType="VARCHAR"/>
<result property="parentId" column="parent_id" jdbcType="VARCHAR"/>
<result property="createTime" column="create_time" jdbcType="TIMESTAMP"/>
<result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
</resultMap>
<sql id="Base_Column_List">
id,category_name,parent_id,
create_time,update_time
</sql>
</mapper>

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.supervision.pdfqaserver.mapper.QuestionHandlerMappingMapper">
<resultMap id="BaseResultMap" type="com.supervision.pdfqaserver.domain.QuestionHandlerMapping">
<id property="id" column="id" jdbcType="VARCHAR"/>
<result property="questionCategoryId" column="question_category_id" jdbcType="VARCHAR"/>
<result property="handler" column="handler" jdbcType="VARCHAR"/>
<result property="createTime" column="create_time" jdbcType="TIMESTAMP"/>
<result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
</resultMap>
<sql id="Base_Column_List">
id,question_category_id,handler,
create_time,update_time
</sql>
</mapper>

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.supervision.pdfqaserver.mapper.TextVectorMapper">
<resultMap id="BaseResultMap" type="com.supervision.pdfqaserver.domain.TextVector">
<result property="id" column="id" jdbcType="VARCHAR"/>
<result property="content" column="content" jdbcType="VARCHAR"/>
<result property="embedding" column="embedding" jdbcType="OTHER" typeHandler="com.supervision.pdfqaserver.config.VectorTypeHandler"/>
<result property="categoryId" column="category_id" jdbcType="VARCHAR"/>
<result property="createTime" column="create_time" jdbcType="TIMESTAMP"/>
<result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
</resultMap>
<sql id="Base_Column_List">
id,content,embedding,
category_id,create_time,update_time
</sql>
<select id="findSimilarByCosine" resultType="com.supervision.pdfqaserver.dto.TextVectorDTO">
SELECT * FROM (
SELECT
id,
content,
embedding,
category_id,
1 - (embedding <![CDATA[<=>]]> #{embedding, typeHandler=com.supervision.pdfqaserver.config.VectorTypeHandler}) AS similarityScore
FROM text_vector
) t
WHERE t.similarityScore > #{threshold}
ORDER BY t.similarityScore DESC
LIMIT #{limit}
</select>
</mapper>

@ -1,21 +1,23 @@
package com.supervision.pdfqaserver;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import com.supervision.pdfqaserver.constant.DocumentContentTypeEnum;
import com.supervision.pdfqaserver.domain.PdfAnalysisOutput;
import com.supervision.pdfqaserver.domain.TextVector;
import com.supervision.pdfqaserver.dto.*;
import com.supervision.pdfqaserver.service.*;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.neo4j.driver.*;
import org.neo4j.driver.Record;
import org.springframework.ai.embedding.Embedding;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
import static org.neo4j.driver.Values.parameters;
@ -206,8 +208,8 @@ class PdfQaServerApplicationTests {
@Test
void testQueryGraph2() {
aiCallService.embedding("");
System.out.println("done");
Embedding embedding = aiCallService.embedding("你好");
System.out.println(embedding.getOutput().length);
}
@ -295,4 +297,44 @@ class PdfQaServerApplicationTests {
List<DomainMetadataDTO> domainMetadataDTOS = domainMetadataService.listByIntentionIds(List.of("1928020787140612097"), "1");
System.out.println(domainMetadataDTOS);
}
@Autowired
private TextVectorService textVectorService;
@Test
public void textVectorTest() {
String texts = """
/
""";
String[] split = texts.split("\n");
List<String> list = Arrays.stream(split).toList();
for (String text : list) {
TextVector textVector = new TextVector();
textVector.setContent(text.trim());
textVector.setCategoryId("查询办公地点");
float[] output = aiCallService.embedding(textVector.getContent()).getOutput();
textVector.setEmbedding(output);
textVectorService.save(textVector);
}
}
@Test
public void textVectorTest2() {
String queryText = "告诉我龙源电力的办公地点?";
float[] output = aiCallService.embedding(queryText).getOutput();
List<TextVectorDTO> similarByCosine = textVectorService.findSimilarByCosine(output, 0.3f, 10);
similarByCosine = similarByCosine.stream().sorted(Comparator.comparingDouble(TextVectorDTO::getSimilarityScore).reversed()).collect(Collectors.toList());
log.info("<<<===========================>>>" );
for (TextVectorDTO vectorDTO : similarByCosine) {
String content = vectorDTO.getContent();
double similarityScore = vectorDTO.getSimilarityScore();
String categoryId = vectorDTO.getCategoryId();
System.out.printf("%s\t%s\t%s\t%s%n",queryText, categoryId , NumberUtil.decimalFormat("0.0000",similarityScore),content);
}
}
}

Loading…
Cancel
Save