diff --git a/pom.xml b/pom.xml
index 34ad1f6..8e4f1ec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -80,6 +80,11 @@
<artifactId>jackson-databind</artifactId>
<version>2.15.3</version>
</dependency>
+ <dependency>
+ <groupId>edu.stanford.nlp</groupId>
+ <artifactId>stanford-corenlp</artifactId>
+ <version>4.5.4</version>
+ </dependency>
</dependencies>
<dependencyManagement>
<dependencies>
diff --git a/src/main/java/com/supervision/pdfqaserver/cache/PromptCache.java b/src/main/java/com/supervision/pdfqaserver/cache/PromptCache.java
index 25e2613..a8e2c8c 100644
--- a/src/main/java/com/supervision/pdfqaserver/cache/PromptCache.java
+++ b/src/main/java/com/supervision/pdfqaserver/cache/PromptCache.java
@@ -10,6 +10,10 @@ public class PromptCache {
public static final String DOERE_TEXT = "DOERE_TEXT";
public static final String DOERE_TABLE = "DOERE_TABLE";
+
+ public static final String CHINESE_TO_ENGLISH = "CHINESE_TO_ENGLISH";
+
+ public static final String ERE_TO_INSERT_CYPHER = "ERE_TO_INSERT_CYPHER";
public static final Map<String, String> promptMap = new HashMap<>();
static {
@@ -18,6 +22,8 @@ public class PromptCache {
private static void init(){
promptMap.put(DOERE_TEXT, DOERE_TEXT_PROMPT);
promptMap.put(DOERE_TABLE, DOERE_TABLE_PROMPT);
+ promptMap.put(CHINESE_TO_ENGLISH, CHINESE_TO_ENGLISH_PROMPT);
+ promptMap.put(ERE_TO_INSERT_CYPHER, ERE_TO_INSERT_CYPHER_PROMPT);
}
@@ -182,4 +188,51 @@ public class PromptCache {
""";
+ private static final String CHINESE_TO_ENGLISH_PROMPT = """
+ 你是一个表格数据处理专家，请严格按以下要求从给出的表格中提取数据：
+
+
+ """;
+
+
+ private static final String ERE_TO_INSERT_CYPHER_PROMPT = """
+ 请将以下三元组数据转换为Neo4j的Cypher语句，要求：
+ 1. **节点**用`(n:Label {name: "Value"})`表示，其中`Label`是实体类型（如`Person`、`Company`）；
+ 2. **关系**用`[r:RELATION_TYPE]`表示，保持与三元组中关系一致；
+ 3. 如果节点或关系已存在，使用`MERGE`避免重复创建；
+ 4. 返回完整的Cypher语句，不要解释。
+
+ ### 输入三元组示例
+ ```json
+ [
+ {"source": "人物","sourceType": "Person", "relation": "创始人", "relationType": "FOUNDED","target": "公司","targetType": "Company"},
+ {"source": "公司","sourceType": "Company ", "relation": "位于", "relationType": "LOCATED_IN","target": "城市","targetType": "City "}
+ ]
+ ```
+
+ ### 输出示例
+
+ MERGE (p:Person {name: "人物"})
+ MERGE (c:Company {name: "公司"})
+ MERGE (city:City {name: "城市"})
+ MERGE (p)-[r1:FOUNDED]->(c)
+ MERGE (c)-[r2:LOCATED_IN]->(city)
+
+ ### 规则补充
+ 1. 实体类型映射：
+ - "人物" → `Person`
+ - "公司" → `Company`
+ - "城市" → `City`
+ 2. 关系类型映射：
+ - "创始人" → `FOUNDED`
+ - "位于" → `LOCATED_IN`
+ 3. 属性统一用`name`字段存储实体名称。
+
+ ### 禁止行为
+ 1. 不要为关系添加属性（除非明确提供）；
+ 2. 不要使用中文标签（如`人物`→`Person`）；
+ 3. 不要省略MERGE的安全约束。
+ ### 请转换以下三元组：
+ {}
+ """;
}
diff --git a/src/main/java/com/supervision/pdfqaserver/constant/DomainMetaGenerationEnum.java b/src/main/java/com/supervision/pdfqaserver/constant/DomainMetaGenerationEnum.java
new file mode 100644
index 0000000..f9c5fd6
--- /dev/null
+++ b/src/main/java/com/supervision/pdfqaserver/constant/DomainMetaGenerationEnum.java
@@ -0,0 +1,23 @@
+package com.supervision.pdfqaserver.constant;
+
+import lombok.Getter;
+
+/**
+ * 域元数据生成枚举类
+ */
+@Getter
+public enum DomainMetaGenerationEnum {
+ // 0=手动录入，1=系统自动
+ DOM_MANUAL_ENTRY("0", "手动录入"),
+
+ SYSTEM_AUTO_GENERATION("1", "系统自动");
+
+
+ private final String code;
+ private final String name;
+
+ DomainMetaGenerationEnum(String code, String name) {
+ this.code = code;
+ this.name = name;
+ }
+}
diff --git a/src/main/java/com/supervision/pdfqaserver/constant/LayoutTypeEnum.java b/src/main/java/com/supervision/pdfqaserver/constant/LayoutTypeEnum.java
new file mode 100644
index 0000000..95edb30
--- /dev/null
+++ b/src/main/java/com/supervision/pdfqaserver/constant/LayoutTypeEnum.java
@@ -0,0 +1,26 @@
+package com.supervision.pdfqaserver.constant;
+
+import lombok.Getter;
+
+@Getter
+public enum LayoutTypeEnum {
+ /**
+ * 文本
+ */
+ TEXT(0, "文本"),
+
+ /**
+ * 表格
+ */
+ TABLE(1, "表格");
+
+
+ private final int code;
+ private final String name;
+
+ LayoutTypeEnum(int code, String name) {
+ this.code = code;
+ this.name = name;
+ }
+
+}
diff --git a/src/main/java/com/supervision/pdfqaserver/domain/DocumentTruncation.java b/src/main/java/com/supervision/pdfqaserver/domain/DocumentTruncation.java
index 4145bc2..7ada473 100644
--- a/src/main/java/com/supervision/pdfqaserver/domain/DocumentTruncation.java
+++ b/src/main/java/com/supervision/pdfqaserver/domain/DocumentTruncation.java
@@ -1,7 +1,6 @@
package com.supervision.pdfqaserver.domain;
import com.baomidou.mybatisplus.annotation.*;
-
import java.io.Serializable;
import java.time.LocalDateTime;
import lombok.Data;
@@ -27,7 +26,7 @@ public class DocumentTruncation implements Serializable {
/**
* 段落id pdf_analysis_output表的id
*/
- private String sectionId;
+ private Integer sectionId;
/**
* 布局类型 0-文本 1-表格
diff --git a/src/main/java/com/supervision/pdfqaserver/dto/DocumentDTO.java b/src/main/java/com/supervision/pdfqaserver/dto/DocumentDTO.java
index c3af439..f05621d 100644
--- a/src/main/java/com/supervision/pdfqaserver/dto/DocumentDTO.java
+++ b/src/main/java/com/supervision/pdfqaserver/dto/DocumentDTO.java
@@ -14,8 +14,9 @@ public class DocumentDTO {
*/
private String id;
- private Integer documentId;
-
+ /**
+ * 段落id
+ */
private Integer sectionId;
private Integer pageNo;
@@ -23,7 +24,7 @@ public class DocumentDTO {
/**
* 内容类型 0：文本 1：表格
*/
- private String layoutType;
+ private Integer layoutType;
/**
* 内容在pdf页面中的顺序，越小表示顺序越靠前
@@ -48,11 +49,9 @@ public class DocumentDTO {
}
public DocumentDTO(PdfAnalysisOutput pdfAnalysisOutput) {
+ this.id = pdfAnalysisOutput.getPdfId().toString();
this.sectionId = pdfAnalysisOutput.getId();
- this.documentId = pdfAnalysisOutput.getPdfId();
- if (null != pdfAnalysisOutput.getLayoutType()) {
- this.layoutType = pdfAnalysisOutput.getLayoutType().toString();
- }
+ this.layoutType = pdfAnalysisOutput.getLayoutType();
this.pageNo = pdfAnalysisOutput.getPageNo();
this.title = pdfAnalysisOutput.getTableTitle();
this.content = pdfAnalysisOutput.getContent();
diff --git a/src/main/java/com/supervision/pdfqaserver/dto/ERAttributeDTO.java b/src/main/java/com/supervision/pdfqaserver/dto/ERAttributeDTO.java
index 5064e7c..8bdd12d 100644
--- a/src/main/java/com/supervision/pdfqaserver/dto/ERAttributeDTO.java
+++ b/src/main/java/com/supervision/pdfqaserver/dto/ERAttributeDTO.java
@@ -1,5 +1,6 @@
package com.supervision.pdfqaserver.dto;
+import com.supervision.pdfqaserver.domain.TruncationErAttribute;
import lombok.Data;
/**
@@ -43,4 +44,14 @@ public class ERAttributeDTO {
this.value = value;
this.dataType = dataType;
}
+
+ public TruncationErAttribute toTruncationErAttribute() {
+ TruncationErAttribute truncationErAttribute = new TruncationErAttribute();
+ truncationErAttribute.setTerId(this.terId);
+ truncationErAttribute.setType(this.type);
+ truncationErAttribute.setAttribute(this.attribute);
+ truncationErAttribute.setValue(this.value);
+ truncationErAttribute.setDataType(this.dataType);
+ return truncationErAttribute;
+ }
}
diff --git a/src/main/java/com/supervision/pdfqaserver/dto/EREDTO.java b/src/main/java/com/supervision/pdfqaserver/dto/EREDTO.java
index 6eda549..cf04e13 100644
--- a/src/main/java/com/supervision/pdfqaserver/dto/EREDTO.java
+++ b/src/main/java/com/supervision/pdfqaserver/dto/EREDTO.java
@@ -4,6 +4,7 @@ import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
+import com.supervision.pdfqaserver.domain.ChineseEnglishWords;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@@ -16,9 +17,9 @@ import java.util.*;
@Data
public class EREDTO {
- private List<EntityExtractionDTO> entities;
+ private List<EntityExtractionDTO> entities = new ArrayList<>();
- private List<RelationExtractionDTO> relations;
+ private List<RelationExtractionDTO> relations = new ArrayList<>();
public EREDTO() {
}
@@ -121,4 +122,21 @@ public class EREDTO {
eredto.setEntities(entities);
return eredto;
}
+
+ public void setEn(List<ChineseEnglishWords> wordsList) {
+ if (CollUtil.isEmpty(wordsList)){
+ return;
+ }
+ for (EntityExtractionDTO entity : entities) {
+ String entityName = entity.getEntity();
+ Optional<ChineseEnglishWords> first = wordsList.stream().filter(w -> StrUtil.equals(w.getChineseWord(), entityName)).findFirst();
+ first.ifPresent(chineseEnglishWords -> entity.setEntityEn(chineseEnglishWords.getEnglishWord()));
+ }
+ for (RelationExtractionDTO relation : relations) {
+ String relationName = relation.getRelation();
+ Optional<ChineseEnglishWords> first = wordsList.stream().filter(w -> StrUtil.equals(w.getChineseWord(), relationName)).findFirst();
+ first.ifPresent(chineseEnglishWords -> relation.setRelationEn(chineseEnglishWords.getEnglishWord()));
+ }
+
+ }
}
diff --git a/src/main/java/com/supervision/pdfqaserver/dto/EntityExtractionDTO.java b/src/main/java/com/supervision/pdfqaserver/dto/EntityExtractionDTO.java
index 30cb118..e976070 100644
--- a/src/main/java/com/supervision/pdfqaserver/dto/EntityExtractionDTO.java
+++ b/src/main/java/com/supervision/pdfqaserver/dto/EntityExtractionDTO.java
@@ -18,6 +18,11 @@ public class EntityExtractionDTO {
*/
private String entity;
+ /**
+ * 实体英文名
+ */
+ private String entityEn;
+
/**
* 实体名
*/
diff --git a/src/main/java/com/supervision/pdfqaserver/dto/RelationExtractionDTO.java b/src/main/java/com/supervision/pdfqaserver/dto/RelationExtractionDTO.java
index 1b35063..f01010d 100644
--- a/src/main/java/com/supervision/pdfqaserver/dto/RelationExtractionDTO.java
+++ b/src/main/java/com/supervision/pdfqaserver/dto/RelationExtractionDTO.java
@@ -1,5 +1,7 @@
package com.supervision.pdfqaserver.dto;
+import com.supervision.pdfqaserver.domain.DomainMetadata;
+import com.supervision.pdfqaserver.domain.TruncationRelationExtraction;
import lombok.Data;
import java.util.List;
@@ -23,11 +25,15 @@ public class RelationExtractionDTO {
*/
private String sourceType;
+ private String sourceTypeEn;
+
/**
*关系
*/
private String relation;
+ private String relationEn;
+
/**
* 尾节点数据
*/
@@ -38,6 +44,8 @@ public class RelationExtractionDTO {
*/
private String targetType;
+ private String targetTypeEn;
+
private List<ERAttributeDTO> attributes;
public RelationExtractionDTO() {
@@ -52,4 +60,23 @@ public class RelationExtractionDTO {
this.sourceType = sourceType;
this.targetType = targetType;
}
+
+ public TruncationRelationExtraction toTruncationRelationExtraction() {
+ TruncationRelationExtraction truncationRelationExtraction = new TruncationRelationExtraction();
+ truncationRelationExtraction.setTruncationId(this.truncationId);
+ truncationRelationExtraction.setSource(this.source);
+ truncationRelationExtraction.setSourceType(this.sourceType);
+ truncationRelationExtraction.setRelation(this.relation);
+ truncationRelationExtraction.setTarget(this.target);
+ truncationRelationExtraction.setTargetType(this.targetType);
+ return truncationRelationExtraction;
+ }
+
+ public DomainMetadata toDomainMetadata() {
+ DomainMetadata domainMetadata = new DomainMetadata();
+ domainMetadata.setSourceType(this.sourceType);
+ domainMetadata.setRelation(this.relation);
+ domainMetadata.setTargetType(this.targetType);
+ return domainMetadata;
+ }
}
diff --git a/src/main/java/com/supervision/pdfqaserver/dto/TruncateDTO.java b/src/main/java/com/supervision/pdfqaserver/dto/TruncateDTO.java
index db4a5d2..1b0ed70 100644
--- a/src/main/java/com/supervision/pdfqaserver/dto/TruncateDTO.java
+++ b/src/main/java/com/supervision/pdfqaserver/dto/TruncateDTO.java
@@ -27,7 +27,7 @@ public class TruncateDTO {
/**
* 段落id pdf_analysis_output表的id
*/
- private String sectionId;
+ private Integer sectionId;
/**
* 表格标题
@@ -40,6 +40,17 @@ public class TruncateDTO {
private String content;
+ public TruncateDTO() {
+ }
+
+ public TruncateDTO(DocumentDTO documentDTO) {
+ this.documentId = Integer.parseInt(documentDTO.getId());
+ this.sectionId = documentDTO.getSectionId();
+ this.layoutType = documentDTO.getLayoutType().toString();
+ this.title = documentDTO.getTitle();
+ this.content = documentDTO.getContent();
+ }
+
public DocumentTruncation toDocumentTruncation() {
DocumentTruncation truncation = new DocumentTruncation();
truncation.setDocumentId(this.documentId);
diff --git a/src/main/java/com/supervision/pdfqaserver/service/ChinesEsToEnglishGenerator.java b/src/main/java/com/supervision/pdfqaserver/service/ChinesEsToEnglishGenerator.java
new file mode 100644
index 0000000..613c206
--- /dev/null
+++ b/src/main/java/com/supervision/pdfqaserver/service/ChinesEsToEnglishGenerator.java
@@ -0,0 +1,6 @@
+package com.supervision.pdfqaserver.service;
+
+public interface ChinesEsToEnglishGenerator {
+
+ String generate(String chinese);
+}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/ChineseEnglishWordsService.java b/src/main/java/com/supervision/pdfqaserver/service/ChineseEnglishWordsService.java
index 3e25515..9c72c1b 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/ChineseEnglishWordsService.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/ChineseEnglishWordsService.java
@@ -2,6 +2,7 @@ package com.supervision.pdfqaserver.service;
import com.supervision.pdfqaserver.domain.ChineseEnglishWords;
import com.baomidou.mybatisplus.extension.service.IService;
+import java.util.List;
/**
* @author Administrator
@@ -10,4 +11,11 @@ import com.baomidou.mybatisplus.extension.service.IService;
*/
public interface ChineseEnglishWordsService extends IService<ChineseEnglishWords> {
+
+ List<ChineseEnglishWords> queryAll();
+
+
+ boolean wordsExists(String word, List<ChineseEnglishWords> wordsList);
+
+ void saveIfNotExists(ChineseEnglishWords words);
}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/DomainMetadataService.java b/src/main/java/com/supervision/pdfqaserver/service/DomainMetadataService.java
index c1f5e1b..59dc4cb 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/DomainMetadataService.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/DomainMetadataService.java
@@ -10,4 +10,6 @@ import com.baomidou.mybatisplus.extension.service.IService;
*/
public interface DomainMetadataService extends IService<DomainMetadata> {
+
+ void saveIfNotExists(DomainMetadata metadata);
}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/TruncationEntityExtractionService.java b/src/main/java/com/supervision/pdfqaserver/service/TruncationEntityExtractionService.java
index 99f4ecc..ab06446 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/TruncationEntityExtractionService.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/TruncationEntityExtractionService.java
@@ -2,6 +2,9 @@ package com.supervision.pdfqaserver.service;
import com.supervision.pdfqaserver.domain.TruncationEntityExtraction;
import com.baomidou.mybatisplus.extension.service.IService;
+import com.supervision.pdfqaserver.dto.EntityExtractionDTO;
+
+import java.util.List;
/**
* @author Administrator
@@ -10,4 +13,5 @@ import com.baomidou.mybatisplus.extension.service.IService;
*/
public interface TruncationEntityExtractionService extends IService<TruncationEntityExtraction> {
+ void saveERE(List<EntityExtractionDTO> entities);
}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/TruncationRelationExtractionService.java b/src/main/java/com/supervision/pdfqaserver/service/TruncationRelationExtractionService.java
index 666e7b7..0ff80e0 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/TruncationRelationExtractionService.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/TruncationRelationExtractionService.java
@@ -2,6 +2,8 @@ package com.supervision.pdfqaserver.service;
import com.supervision.pdfqaserver.domain.TruncationRelationExtraction;
import com.baomidou.mybatisplus.extension.service.IService;
+import com.supervision.pdfqaserver.dto.RelationExtractionDTO;
+import java.util.List;
/**
* @author Administrator
@@ -10,4 +12,5 @@ import com.baomidou.mybatisplus.extension.service.IService;
*/
public interface TruncationRelationExtractionService extends IService<TruncationRelationExtraction> {
+ void saveERE(List<RelationExtractionDTO> relations);
}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/impl/ChinesEsToEnglishGeneratorImpl.java b/src/main/java/com/supervision/pdfqaserver/service/impl/ChinesEsToEnglishGeneratorImpl.java
new file mode 100644
index 0000000..adefdcf
--- /dev/null
+++ b/src/main/java/com/supervision/pdfqaserver/service/impl/ChinesEsToEnglishGeneratorImpl.java
@@ -0,0 +1,24 @@
+package com.supervision.pdfqaserver.service.impl;
+
+import com.supervision.pdfqaserver.cache.PromptCache;
+import com.supervision.pdfqaserver.service.ChinesEsToEnglishGenerator;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.ai.ollama.OllamaChatModel;
+import org.springframework.stereotype.Service;
+import static com.supervision.pdfqaserver.cache.PromptCache.CHINESE_TO_ENGLISH;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class ChinesEsToEnglishGeneratorImpl implements ChinesEsToEnglishGenerator {
+
+ private final OllamaChatModel ollamaChatModel;
+ @Override
+ public String generate(String chinese) {
+ log.info("generate:开始翻译: {}",chinese);
+ String prompt = PromptCache.promptMap.get(CHINESE_TO_ENGLISH);
+ ollamaChatModel.call("请将以下中文翻译成英文: " + chinese);
+ return null;
+ }
+}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/impl/ChineseEnglishWordsServiceImpl.java b/src/main/java/com/supervision/pdfqaserver/service/impl/ChineseEnglishWordsServiceImpl.java
index 57106ef..70ca89d 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/impl/ChineseEnglishWordsServiceImpl.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/impl/ChineseEnglishWordsServiceImpl.java
@@ -1,10 +1,12 @@
package com.supervision.pdfqaserver.service.impl;
+import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.pdfqaserver.domain.ChineseEnglishWords;
import com.supervision.pdfqaserver.service.ChineseEnglishWordsService;
import com.supervision.pdfqaserver.mapper.ChineseEnglishWordsMapper;
import org.springframework.stereotype.Service;
+import java.util.List;
/**
* @author Administrator
@@ -15,6 +17,26 @@ import org.springframework.stereotype.Service;
public class ChineseEnglishWordsServiceImpl extends ServiceImpl<ChineseEnglishWordsMapper, ChineseEnglishWords>
implements ChineseEnglishWordsService{
+ @Override
+ public List<ChineseEnglishWords> queryAll() {
+ return this.list();
+ }
+
+ @Override
+ public boolean wordsExists(String word, List<ChineseEnglishWords> wordsList) {
+ if (StrUtil.isEmpty(word)){
+ return true;
+ }
+ return wordsList.stream().anyMatch(w->StrUtil.equals(w.getChineseWord(),word));
+ }
+
+ @Override
+ public void saveIfNotExists(ChineseEnglishWords words) {
+ boolean exists = this.lambdaQuery().eq(ChineseEnglishWords::getChineseWord, words.getChineseWord()).exists();
+ if (!exists){
+ this.save(words);
+ }
+ }
}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/impl/DomainMetadataServiceImpl.java b/src/main/java/com/supervision/pdfqaserver/service/impl/DomainMetadataServiceImpl.java
index d246477..2f4219b 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/impl/DomainMetadataServiceImpl.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/impl/DomainMetadataServiceImpl.java
@@ -15,6 +15,17 @@ import org.springframework.stereotype.Service;
public class DomainMetadataServiceImpl extends ServiceImpl<DomainMetadataMapper, DomainMetadata>
implements DomainMetadataService{
+ @Override
+ public void saveIfNotExists(DomainMetadata metadata) {
+
+ boolean exists = this.lambdaQuery()
+ .eq(DomainMetadata::getSourceType, metadata.getSourceType())
+ .eq(DomainMetadata::getTargetType, metadata.getTargetType())
+ .eq(DomainMetadata::getRelation, metadata.getRelation()).exists();
+ if (!exists) {
+ this.save(metadata);
+ }
+ }
}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/impl/KnowledgeGraphServiceImpl.java b/src/main/java/com/supervision/pdfqaserver/service/impl/KnowledgeGraphServiceImpl.java
index 06364c7..fac2c4a 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/impl/KnowledgeGraphServiceImpl.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/impl/KnowledgeGraphServiceImpl.java
@@ -1,15 +1,16 @@
package com.supervision.pdfqaserver.service.impl;
import cn.hutool.core.collection.CollUtil;
-import com.supervision.pdfqaserver.dto.EREDTO;
+import cn.hutool.core.util.StrUtil;
+import com.supervision.pdfqaserver.constant.DomainMetaGenerationEnum;
+import com.supervision.pdfqaserver.domain.ChineseEnglishWords;
+import com.supervision.pdfqaserver.domain.DomainMetadata;
+import com.supervision.pdfqaserver.dto.*;
import com.supervision.pdfqaserver.domain.PdfAnalysisOutput;
-import com.supervision.pdfqaserver.dto.DocumentDTO;
-import com.supervision.pdfqaserver.dto.TruncateDTO;
import com.supervision.pdfqaserver.service.*;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
-
import java.util.ArrayList;
import java.util.List;
@@ -30,13 +31,11 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
private final PdfAnalysisOutputService pdfAnalysisOutputService;
- private final PdfInfoService pdfInfoService;
-
private final TruncationEntityExtractionService truncationEntityExtractionService;
private final TruncationRelationExtractionService relationExtractionService;
- private final TruncationErAttributeService truncationErAttributeService;
+ private final ChinesEsToEnglishGenerator chinesEsToEnglishGenerator;
@Override
public void generateGraph(String documentId) {
@@ -55,6 +54,9 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
List<EREDTO> eredtoList = new ArrayList<>();
for (TruncateDTO truncateDTO : truncateDTOS) {
EREDTO eredto = tripleConversionPipeline.doEre(truncateDTO);
+ if (null == eredto){
+ continue;
+ }
// 保存实体关系抽取结果
this.saveERE(eredto, truncateDTO.getId());
}
@@ -62,13 +64,63 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
// 合并实体关系抽取结果
List<EREDTO> mergedList = tripleConversionPipeline.mergeEreResults(eredtoList);
+ // 保存领域元数据
for (EREDTO eredto : mergedList) {
- String insertCypher = tripleToCypherExecutor.generateInsertCypher(eredto);
+ List<RelationExtractionDTO> relations = eredto.getRelations();
+ if (CollUtil.isEmpty(relations)){
+ continue;
+ }
+ for (RelationExtractionDTO relation : relations) {
+ DomainMetadata domainMetadata = relation.toDomainMetadata();
+ domainMetadata.setDomainType("1");
+ domainMetadata.setGenerationType(DomainMetaGenerationEnum.SYSTEM_AUTO_GENERATION.getCode());
+ domainMetadataService.saveIfNotExists(domainMetadata);
+ }
+ }
+ // 保存字典
+ List<ChineseEnglishWords> allWords = chineseEnglishWordsService.queryAll();
+ for (EREDTO eredto : mergedList) {
+ List<EntityExtractionDTO> entities = eredto.getEntities();
+ if (CollUtil.isNotEmpty(entities)){
+ for (EntityExtractionDTO entityDTO : entities) {
+ saveWordsIfNecessary(entityDTO.getEntity(), allWords);
+ }
+ }
+ List<RelationExtractionDTO> relations = eredto.getRelations();
+ if (CollUtil.isNotEmpty(relations)){
+ for (RelationExtractionDTO relationDTO : relations) {
+ saveWordsIfNecessary(relationDTO.getRelation(), allWords);
+ }
+ }
+ }
+
+ // 生成cypher语句
+ for (EREDTO eredto : mergedList) {
+ eredto.setEn(allWords);
+ String insertCypher = tripleToCypherExecutor.generateInsertCypher(eredto);
+ log.info("insertCypher:{}", insertCypher);
tripleToCypherExecutor.executeCypher(insertCypher);
}
+ }
+
+ private void saveWordsIfNecessary(String word, List<ChineseEnglishWords> allWords) {
+ boolean exists = chineseEnglishWordsService.wordsExists(word, allWords);
+ if (exists){
+ return;
+ }
+ String generate = chinesEsToEnglishGenerator.generate(word);
+ if (StrUtil.isEmpty(generate)){
+ log.info("生成英文名称失败，entity:{}", word);
+ return;
+ }
+ ChineseEnglishWords words = new ChineseEnglishWords();
+ words.setChineseWord(word);
+ words.setEnglishWord(generate);
+ chineseEnglishWordsService.saveIfNotExists(words);
+ allWords.add(words);// 更新缓存
}
@Override
@@ -79,6 +131,11 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
@Override
public void saveERE(EREDTO eredto, String truncationId) {
+ // 保存实体信息
+ truncationEntityExtractionService.saveERE(eredto.getEntities());
+
+ // 保存关系
+ relationExtractionService.saveERE(eredto.getRelations());
}
}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/impl/TripleConversionPipelineImpl.java b/src/main/java/com/supervision/pdfqaserver/service/impl/TripleConversionPipelineImpl.java
index 57899ca..5c4ef14 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/impl/TripleConversionPipelineImpl.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/impl/TripleConversionPipelineImpl.java
@@ -3,15 +3,19 @@ package com.supervision.pdfqaserver.service.impl;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import com.supervision.pdfqaserver.cache.PromptCache;
+import com.supervision.pdfqaserver.constant.LayoutTypeEnum;
import com.supervision.pdfqaserver.dto.*;
import com.supervision.pdfqaserver.service.TripleConversionPipeline;
+import edu.stanford.nlp.pipeline.CoreDocument;
+import edu.stanford.nlp.pipeline.CoreSentence;
+import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.ollama.OllamaChatModel;
import org.springframework.stereotype.Service;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
+
@Slf4j
@Service
@RequiredArgsConstructor
@@ -31,7 +35,38 @@ public class TripleConversionPipelineImpl implements TripleConversionPipeline {
return Integer.compare(o1.getPageNo(), o2.getPageNo());
}
).toList();
- return null;
+
+ Properties props = new Properties();
+ props.setProperty("annotators", "tokenize, ssplit");
+ // 创建管道
+ StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
+ List<TruncateDTO> truncateDTOS = new ArrayList<>();
+ for (DocumentDTO documentDTO : documentDTOList) {
+ String content = documentDTO.getContent();
+ if (StrUtil.isEmpty(content)){
+ continue;
+ }
+ Integer layoutType = documentDTO.getLayoutType();
+ if (LayoutTypeEnum.TEXT.getCode() == layoutType){
+ // 如果是文本类型的布局，进行合并
+ CoreDocument document = new CoreDocument(content);
+ // 分析文本
+ pipeline.annotate(document);
+ // 获取句子
+ for (CoreSentence sentence : document.sentences()) {
+ TruncateDTO truncateDTO = new TruncateDTO(documentDTO);
+ truncateDTO.setContent(sentence.text());
+ truncateDTOS.add(truncateDTO);
+ }
+ } else if (LayoutTypeEnum.TABLE.getCode() == layoutType) {
+ // 如果是表格类型的布局，直接添加到列表中
+ TruncateDTO truncateDTO = new TruncateDTO(documentDTO);
+ truncateDTOS.add(truncateDTO);
+ } else {
+ log.info("sliceDocuments:错误的布局类型: {}", layoutType);
+ }
+ }
+ return truncateDTOS;
}
@Override
@@ -39,13 +74,11 @@ public class TripleConversionPipelineImpl implements TripleConversionPipeline {
if (StrUtil.equals(truncateDTO.getLayoutType(),"0")){
- EREDTO eredto = doTextEre(truncateDTO);
- return eredto;
+ return doTextEre(truncateDTO);
}
if (StrUtil.equals(truncateDTO.getLayoutType(),"1")){
- EREDTO eredto = doTableEre(truncateDTO);
- return eredto;
+ return doTableEre(truncateDTO);
}
log.info("doEre:错误的布局类型: {}", truncateDTO.getLayoutType());
return null;
@@ -80,26 +113,119 @@ public class TripleConversionPipelineImpl implements TripleConversionPipeline {
if (CollUtil.isEmpty(eredtoList)){
return merged;
}
+ Map<String, EntityExtractionDTO> entityMap = new HashMap<>();
+ Map<String, RelationExtractionDTO> relationMap = new HashMap<>();
for (EREDTO eredto : eredtoList) {
List<EntityExtractionDTO> entities = eredto.getEntities();
if (CollUtil.isNotEmpty(entities)){
for (EntityExtractionDTO entity : entities) {
- String e = entity.getEntity();
- String name = entity.getName();
- // entity.getEntity() 和 entity.getName() 完全相等看作是同一个数据
+ String key = generateEntityMapKey(entity);
+ mergeAttribute(entityMap,entity, key);
}
}
List<RelationExtractionDTO> relations = eredto.getRelations();
if (CollUtil.isNotEmpty(relations)){
for (RelationExtractionDTO relation : relations) {
- String source = relation.getSource();
- String target = relation.getTarget();
- String re = relation.getRelation();
// source和target,re完全相等看作是同一个数据
+ String relationMapKey = generateRelationMapKey(relation);
+ mergeAttribute(relationMap,relation, relationMapKey);
}
}
}
+ // 利用合并后的map生成新的EREDTO
+ // 优先先把有关系的节点与关系组合在一次
+ Set<String> relationEntityKey = new HashSet<>();
+ for (Map.Entry<String, RelationExtractionDTO> relationEntry : relationMap.entrySet()) {
+ RelationExtractionDTO value = relationEntry.getValue();
+ EntityExtractionDTO sourceEntity = entityMap.get(StrUtil.join("_", value.getSourceType(), value.getSource()));
+ if (null == sourceEntity){
+ log.warn("mergeEreResults:根据entity:{},name:{}未在entityMap中找到头节点映射关系", value.getSourceType(), value.getSource());
+ continue;
+ }
+ EntityExtractionDTO targetEntity = entityMap.get(StrUtil.join("_", value.getTargetType(), value.getTarget()));
+ if (null == targetEntity){
+ log.warn("mergeEreResults:根据entity:{},name:{}未在entityMap中找到尾节点映射关系", value.getTargetType(), value.getTarget());
+ continue;
+ }
+ EREDTO eredto = new EREDTO();
+ eredto.setEntities(List.of(sourceEntity,targetEntity));
+ eredto.setRelations(List.of(value));
+ merged.add(eredto);
+ relationEntityKey.addAll(List.of(generateEntityMapKey(sourceEntity),generateEntityMapKey(targetEntity)));
+ }
+ // 将没有关系的节点单独放在一起
+ List<EntityExtractionDTO> leavedEntities = new ArrayList<>();
+ for (Map.Entry<String, EntityExtractionDTO> entry : entityMap.entrySet()) {
+ if (!relationEntityKey.contains(entry.getKey())){
+ leavedEntities.add(entry.getValue());
+ }
+ }
+ EREDTO eredto = new EREDTO();
+ eredto.setEntities(leavedEntities);
+ merged.add(eredto);
+ return merged;
+ }
- return null;
+ private void mergeAttribute(Map<String, RelationExtractionDTO> entityMap,RelationExtractionDTO relation, String key) {
+
+ RelationExtractionDTO cachedEntity = entityMap.get(key);
+ if (null == cachedEntity){
+ entityMap.put(key, relation);
+ }else {
+ if (CollUtil.isEmpty(relation.getAttributes())){
+ return;
+ }
+ // 合并属性
+ List<ERAttributeDTO> attributes = relation.getAttributes();
+ if (null == attributes){
+ attributes = new ArrayList<>();
+ }
+ for (ERAttributeDTO attribute : relation.getAttributes()) {
+ String attributeKey = attribute.getAttribute();
+ String attributeValue = attribute.getValue();
+ if (StrUtil.isEmpty(attributeKey) || StrUtil.isEmpty(attributeValue)){
+ continue;
+ }
+ // 如果属性已经存在，则不添加
+ if (attributes.stream().noneMatch(a -> StrUtil.equals(a.getAttribute(), attributeKey))) {
+ attributes.add(attribute);
+ }
+ }
+ }
+ }
+ private void mergeAttribute(Map<String, EntityExtractionDTO> entityMap,EntityExtractionDTO entity, String key) {
+
+ EntityExtractionDTO cachedEntity = entityMap.get(key);
+ if (null == cachedEntity){
+ entityMap.put(key, entity);
+ }else {
+ if (CollUtil.isEmpty(entity.getAttributes())){
+ return;
+ }
+ // 合并属性
+ List<ERAttributeDTO> attributes = entity.getAttributes();
+ if (null == attributes){
+ attributes = new ArrayList<>();
+ }
+ for (ERAttributeDTO attribute : entity.getAttributes()) {
+ String attributeKey = attribute.getAttribute();
+ String attributeValue = attribute.getValue();
+ if (StrUtil.isEmpty(attributeKey) || StrUtil.isEmpty(attributeValue)){
+ continue;
+ }
+ // 如果属性已经存在，则不添加
+ if (attributes.stream().noneMatch(a -> StrUtil.equals(a.getAttribute(), attributeKey))) {
+ attributes.add(attribute);
+ }
+ }
+ }
+ }
+
+ private String generateEntityMapKey(EntityExtractionDTO entityExtractionDTO) {
+ return entityExtractionDTO.getEntity() + "_" + entityExtractionDTO.getName();
+ }
+
+ private String generateRelationMapKey(RelationExtractionDTO relationExtractionDTO) {
+ return relationExtractionDTO.getSource() + "_" + relationExtractionDTO.getTarget() + "_" + relationExtractionDTO.getRelation();
}
}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/impl/TripleToCypherExecutorImpl.java b/src/main/java/com/supervision/pdfqaserver/service/impl/TripleToCypherExecutorImpl.java
index 1d82afa..025fee0 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/impl/TripleToCypherExecutorImpl.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/impl/TripleToCypherExecutorImpl.java
@@ -1,11 +1,13 @@
package com.supervision.pdfqaserver.service.impl;
+import com.supervision.pdfqaserver.cache.PromptCache;
import com.supervision.pdfqaserver.dto.EREDTO;
import com.supervision.pdfqaserver.service.TripleToCypherExecutor;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.ollama.OllamaChatModel;
import org.springframework.stereotype.Service;
+import static com.supervision.pdfqaserver.cache.PromptCache.ERE_TO_INSERT_CYPHER;
@Slf4j
@Service
@@ -15,7 +17,10 @@ public class TripleToCypherExecutorImpl implements TripleToCypherExecutor {
private final OllamaChatModel ollamaChatModel;
@Override
public String generateInsertCypher(EREDTO eredto) {
- return null;
+
+ String prompt = PromptCache.promptMap.get(ERE_TO_INSERT_CYPHER);
+ String call = ollamaChatModel.call(prompt);
+ return call;
}
@Override
diff --git a/src/main/java/com/supervision/pdfqaserver/service/impl/TruncationEntityExtractionServiceImpl.java b/src/main/java/com/supervision/pdfqaserver/service/impl/TruncationEntityExtractionServiceImpl.java
index aafd16f..f80dc1f 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/impl/TruncationEntityExtractionServiceImpl.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/impl/TruncationEntityExtractionServiceImpl.java
@@ -1,20 +1,54 @@
package com.supervision.pdfqaserver.service.impl;
+import cn.hutool.core.collection.CollUtil;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.supervision.pdfqaserver.domain.TruncationEntityExtraction;
+import com.supervision.pdfqaserver.domain.TruncationErAttribute;
+import com.supervision.pdfqaserver.dto.ERAttributeDTO;
+import com.supervision.pdfqaserver.dto.EntityExtractionDTO;
import com.supervision.pdfqaserver.service.TruncationEntityExtractionService;
import com.supervision.pdfqaserver.mapper.TruncationEntityExtractionMapper;
+import com.supervision.pdfqaserver.service.TruncationErAttributeService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
+import java.util.List;
+
/**
* @author Administrator
* @description 针对表【truncation_entity_extraction(片段实体抽取)】的数据库操作Service实现
* @createDate 2025-04-27 11:45:24
*/
+@Slf4j
@Service
+@RequiredArgsConstructor
public class TruncationEntityExtractionServiceImpl extends ServiceImpl<TruncationEntityExtractionMapper, TruncationEntityExtraction>
implements TruncationEntityExtractionService{
+ private final TruncationErAttributeService truncationErAttributeService;
+ @Override
+ public void saveERE(List<EntityExtractionDTO> entities) {
+ if (CollUtil.isEmpty(entities)){
+ return;
+ }
+ for (EntityExtractionDTO entity : entities) {
+ TruncationEntityExtraction tee = new TruncationEntityExtraction();
+ tee.setTruncationId(entity.getTruncationId());
+ tee.setEntity(entity.getEntity());
+ tee.setName(entity.getName());
+ this.save(tee);
+ List<ERAttributeDTO> attributes = entity.getAttributes();
+ if (CollUtil.isEmpty(attributes)){
+ continue;
+ }
+ for (ERAttributeDTO attribute : attributes) {
+ attribute.setTerId(tee.getId());
+ TruncationErAttribute era = attribute.toTruncationErAttribute();
+ truncationErAttributeService.save(era);
+ }
+ }
+ }
}
diff --git a/src/main/java/com/supervision/pdfqaserver/service/impl/TruncationRelationExtractionServiceImpl.java b/src/main/java/com/supervision/pdfqaserver/service/impl/TruncationRelationExtractionServiceImpl.java
index 5ab6692..87b3ec3 100644
--- a/src/main/java/com/supervision/pdfqaserver/service/impl/TruncationRelationExtractionServiceImpl.java
+++ b/src/main/java/com/supervision/pdfqaserver/service/impl/TruncationRelationExtractionServiceImpl.java
@@ -1,20 +1,51 @@
package com.supervision.pdfqaserver.service.impl;
+import cn.hutool.core.collection.CollUtil;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.supervision.pdfqaserver.domain.TruncationErAttribute;
import com.supervision.pdfqaserver.domain.TruncationRelationExtraction;
+import com.supervision.pdfqaserver.dto.ERAttributeDTO;
+import com.supervision.pdfqaserver.dto.RelationExtractionDTO;
+import com.supervision.pdfqaserver.service.TruncationErAttributeService;
import com.supervision.pdfqaserver.service.TruncationRelationExtractionService;
import com.supervision.pdfqaserver.mapper.TruncationRelationExtractionMapper;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
+import java.util.List;
+
/**
* @author Administrator
* @description 针对表【truncation_relation_extraction(片段关系抽取)】的数据库操作Service实现
* @createDate 2025-04-27 11:45:24
*/
+@Slf4j
@Service
+@RequiredArgsConstructor
public class TruncationRelationExtractionServiceImpl extends ServiceImpl<TruncationRelationExtractionMapper, TruncationRelationExtraction>
implements TruncationRelationExtractionService{
+ private final TruncationErAttributeService truncationErAttributeService;
+ @Override
+ public void saveERE(List<RelationExtractionDTO> relations) {
+ if (CollUtil.isEmpty(relations)){
+ return;
+ }
+
+ for (RelationExtractionDTO relation : relations) {
+ TruncationRelationExtraction re = relation.toTruncationRelationExtraction();
+ this.save(re);
+ if (CollUtil.isEmpty(relation.getAttributes())){
+ continue;
+ }
+ for (ERAttributeDTO attribute : relation.getAttributes()) {
+ TruncationErAttribute era = attribute.toTruncationErAttribute();
+ era.setTerId(re.getId());
+ truncationErAttributeService.save(era);
+ }
+ }
+ }
}
diff --git a/src/main/resources/mapper/DocumentTruncationMapper.xml b/src/main/resources/mapper/DocumentTruncationMapper.xml
index 2440ce9..d32cbb8 100644
--- a/src/main/resources/mapper/DocumentTruncationMapper.xml
+++ b/src/main/resources/mapper/DocumentTruncationMapper.xml
@@ -7,7 +7,7 @@
<resultMap id="BaseResultMap" type="com.supervision.pdfqaserver.domain.DocumentTruncation">
<id property="id" column="id" jdbcType="VARCHAR"/>
<result property="documentId" column="document_id" jdbcType="INTEGER"/>
- <result property="sectionId" column="section_id" jdbcType="VARCHAR"/>
+ <result property="sectionId" column="section_id" jdbcType="INTEGER"/>
<result property="layoutType" column="layout_type" jdbcType="VARCHAR"/>
<result property="title" column="title" jdbcType="VARCHAR"/>
<result property="content" column="content" jdbcType="VARCHAR"/>