领域元数据提取中..

src/main/java/com/supervision/pdfqaserver/constant/DocumentContentTypeEnum.java

@@ -0,0 +1,35 @@
+package com.supervision.pdfqaserver.constant;
+
+import lombok.Getter;
+
+/**
+ * 文档内容类型
+ */
+@Getter
+public enum DocumentContentTypeEnum {
+
+    /**
+     * 研报
+     */
+    REPORT("0", "研报"),
+
+    /**
+     * 对话
+     */
+    DIALOGUE("1", "对话"),
+
+    /**
+     * 记录
+     */
+    RECORD("2", "记录");
+
+
+    DocumentContentTypeEnum(String type, String desc) {
+        this.type = type;
+        this.desc = desc;
+    }
+
+    private final String type;
+
+    private final String desc;
+}

src/main/java/com/supervision/pdfqaserver/domain/ChineseEnglishWords.java

@@ -10,6 +10,7 @@ import lombok.Data;
  * 中英文对照字典
  * @TableName chinese_english_words
  */
+@Deprecated(since = "v_0.0.2")
 @TableName(value ="chinese_english_words")
 @Data
 public class ChineseEnglishWords implements Serializable {

src/main/java/com/supervision/pdfqaserver/domain/ErAttribute.java

@@ -43,10 +43,6 @@ public class ErAttribute implements Serializable {
      */
     private String erType;
 
-    /**
-     * 
-     */
-    private String parentId;
 
     /**
      * 创建时间

src/main/java/com/supervision/pdfqaserver/domain/Intention.java

@@ -6,8 +6,8 @@ import java.time.LocalDateTime;
 import lombok.Data;
 
 /**
- * 
- * @TableName intention
+ *  一个digest 和一个 domainCategoryId 组成一个意图
+ * @TableName intention 意图表
  */
 @TableName(value ="intention")
 @Data
@@ -33,6 +33,12 @@ public class Intention implements Serializable {
      */
     private String domainCategoryId;
 
+
+    /**
+     * 数据来源：0=手动录入，1=系统自动
+     */
+    private String generationType;
+
     /**
      * 创建时间
      */

src/main/java/com/supervision/pdfqaserver/domain/PdfInfo.java

@@ -56,7 +56,7 @@ public class PdfInfo implements Serializable {
 
 
     /**
-     * 训练状态 0:未训练 1：训练成功 2：训练失败
+     * 训练状态 0:开始训练 1：训练成功 2：训练失败
      */
     private Integer trainStatus;
 

src/main/java/com/supervision/pdfqaserver/dto/DomainMetadataDTO.java

@@ -1,35 +1,63 @@
 package com.supervision.pdfqaserver.dto;
 
+import com.supervision.pdfqaserver.domain.DomainMetadata;
 import lombok.Data;
+import java.util.List;
 
+/**
+ * 领域元数据DTO
+ */
 @Data
 public class DomainMetadataDTO {
 
     private String id;
 
     /**
-     * 领域类型
+     * 意图摘要
      */
-    private String domainType;
+    private String intentDigest;
+
+
+    /**
+     * 领域分类id
+     */
+    private String domainCategoryId;
 
     /**
      * 头节点类型
      */
     private String sourceType;
 
+    private List<ERAttributeDTO> sourceAttributes;
+
     /**
      * 关系
      */
     private String relation;
 
+    private List<ERAttributeDTO> relationAttributes;
+
     /**
      * 尾节点类型
      */
     private String targetType;
 
+    private List<ERAttributeDTO> targetAttributes;
+
     /**
      * 数据来源：0=手动录入，1=系统自动
      */
     private String generationType;
 
+    public DomainMetadata toDomainMetadata() {
+        DomainMetadata domainMetadata = new DomainMetadata();
+        domainMetadata.setId(this.id);
+        domainMetadata.setSourceType(this.sourceType);
+        domainMetadata.setTargetType(this.targetType);
+        domainMetadata.setRelation(this.relation);
+        domainMetadata.setDomainCategoryId(this.domainCategoryId);
+        domainMetadata.setGenerationType(this.generationType);
+        return domainMetadata;
+    }
+
 }

src/main/java/com/supervision/pdfqaserver/dto/ERAttributeDTO.java

@@ -1,68 +1,34 @@
 package com.supervision.pdfqaserver.dto;
 
-import com.supervision.pdfqaserver.domain.TruncationErAttribute;
 import lombok.Data;
 
-/**
- * 实体属性
- */
 @Data
 public class ERAttributeDTO {
 
     private String id;
 
     /**
-     * 片段实体属性表 既可以是truncation_entity_extraction表id也可以是truncation_relation_extraction表id
+     * 领域分类id
      */
-    private String terId;
+    private String domainMetadataId;
 
     /**
-     * 类型 0：terId关联的id为实体 1：terId关联的id为关系
+     * 属性名
      */
-    private String associationType;
+    private String erName;
 
     /**
-     * 属性名
+     * 属性值类型
      */
-    private String attribute;
-
-    private String attributeEn;
+    private String attrName;
 
     /**
-     * 属性值
+     * 属性值类型
      */
-    private String value;
+    private String attrValueType;
 
     /**
-     * 数据类型 0：字符串 1：数字
+     * 节点 1 关系 2
      */
-    private String dataType;
-
-    public ERAttributeDTO() {
-    }
-
-    public ERAttributeDTO(TruncationErAttribute truncationErAttribute) {
-        this.id = truncationErAttribute.getId();
-        this.terId = truncationErAttribute.getTerId();
-        this.associationType = truncationErAttribute.getAssociationType();
-        this.attribute = truncationErAttribute.getAttribute();
-        this.value = truncationErAttribute.getValue();
-        this.dataType = truncationErAttribute.getDataType();
-    }
-
-    public ERAttributeDTO(String attribute, String value, String dataType) {
-        this.attribute = attribute;
-        this.value = value;
-        this.dataType = dataType;
-    }
-
-    public TruncationErAttribute toTruncationErAttribute() {
-        TruncationErAttribute truncationErAttribute = new TruncationErAttribute();
-        truncationErAttribute.setTerId(this.terId);
-        truncationErAttribute.setAssociationType(this.associationType);
-        truncationErAttribute.setAttribute(this.attribute);
-        truncationErAttribute.setValue(this.value);
-        truncationErAttribute.setDataType(this.dataType);
-        return truncationErAttribute;
-    }
+    private String erType;
 }

src/main/java/com/supervision/pdfqaserver/dto/EREDTO.java

@@ -43,16 +43,17 @@ public class EREDTO {
                 name = StrUtil.trim(name);
                 type = StrUtil.trim(type);
                 JSONObject attributes = nodeJson.getJSONObject("attributes");
-                List<ERAttributeDTO> erAttributeDTOS = new ArrayList<>();
+                List<TruncationERAttributeDTO> truncationErAttributeDTOS = new ArrayList<>();
                 if (CollUtil.isNotEmpty(attributes)){
                     for (String key : attributes.keySet()) {
                         Object value = attributes.get(key);
                         String valueString = attributes.getString(key);
-                        ERAttributeDTO erAttributeDTO = new ERAttributeDTO(key, valueString, value instanceof Number?"1":"0");
-                        erAttributeDTOS.add(erAttributeDTO);
+                        TruncationERAttributeDTO truncationErAttributeDTO = new TruncationERAttributeDTO(key, valueString, value instanceof Number?"1":"0");
+                        truncationErAttributeDTO.setAssociationType("0");
+                        truncationErAttributeDTOS.add(truncationErAttributeDTO);
                     }
                 }
-                EntityExtractionDTO entityExtraction = new EntityExtractionDTO(truncationId,type,name, erAttributeDTOS);
+                EntityExtractionDTO entityExtraction = new EntityExtractionDTO(truncationId,type,name, truncationErAttributeDTOS);
                 entities.add(entityExtraction);
             }
         }
@@ -63,7 +64,7 @@ public class EREDTO {
                 String target = relationJson.getString("target");
                 String type = relationJson.getString("type");
                 JSONObject attributes = relationJson.getJSONObject("attributes");
-                List<ERAttributeDTO> erAttributeDTOS = new ArrayList<>();
+                List<TruncationERAttributeDTO> truncationErAttributeDTOS = new ArrayList<>();
                 if (CollUtil.isNotEmpty(attributes)){
                     for (String key : attributes.keySet()) {
                         if (StrUtil.isBlank(key)){
@@ -77,8 +78,9 @@ public class EREDTO {
                             value = StrUtil.trim((String) value);
                         }
                         String valueString = attributes.getString(key);
-                        ERAttributeDTO erAttributeDTO = new ERAttributeDTO(key, valueString, value instanceof Number?"1":"0");
-                        erAttributeDTOS.add(erAttributeDTO);
+                        TruncationERAttributeDTO truncationErAttributeDTO = new TruncationERAttributeDTO(key, valueString, value instanceof Number?"1":"0");
+                        truncationErAttributeDTO.setAssociationType("1");
+                        truncationErAttributeDTOS.add(truncationErAttributeDTO);
                     }
                 }
                 if (StrUtil.isBlank(source) || StrUtil.isBlank(target)){
@@ -98,7 +100,7 @@ public class EREDTO {
                     continue;
                 }
                 RelationExtractionDTO relationExtractionDTO = new RelationExtractionDTO(truncationId,source,
-                        sourceTypeOpt.get().getEntity(),type,target,targetTypeOpt.get().getEntity(), erAttributeDTOS);
+                        sourceTypeOpt.get().getEntity(),type,target,targetTypeOpt.get().getEntity(), truncationErAttributeDTOS);
                 relationsList.add(relationExtractionDTO);
             }
         }
@@ -127,7 +129,7 @@ public class EREDTO {
             // 避免表格行名重复
             entityExtractionDTO.setName("行-" +  RandomUtil.randomString(UUID.randomUUID().toString(), 10));
             entityExtractionDTO.setTruncationId(truncationId);
-            List<ERAttributeDTO> erAttributeDTOS = new ArrayList<>();
+            List<TruncationERAttributeDTO> truncationErAttributeDTOS = new ArrayList<>();
             for (Map.Entry<String, Object> tableEntry : tableJson.entrySet()) {
                 String key = tableEntry.getKey();
                 if (StrUtil.isBlank(key)){
@@ -141,10 +143,10 @@ public class EREDTO {
                     }
                     value = StrUtil.trim((String) value);
                 }
-                ERAttributeDTO erAttributeDTO = new ERAttributeDTO(key, value.toString(), value instanceof Number ? "1" : "0");
-                erAttributeDTOS.add(erAttributeDTO);
+                TruncationERAttributeDTO truncationErAttributeDTO = new TruncationERAttributeDTO(key, value.toString(), value instanceof Number ? "1" : "0");
+                truncationErAttributeDTOS.add(truncationErAttributeDTO);
             }
-            entityExtractionDTO.setAttributes(erAttributeDTOS);
+            entityExtractionDTO.setAttributes(truncationErAttributeDTOS);
             entities.add(entityExtractionDTO);
         }
         eredto.setEntities(entities);
@@ -161,7 +163,7 @@ public class EREDTO {
             first.ifPresent(chineseEnglishWords -> entity.setEntityEn(chineseEnglishWords.getEnglishWord()));
 
             if (CollUtil.isNotEmpty(entity.getAttributes())){
-                for (ERAttributeDTO attribute : entity.getAttributes()) {
+                for (TruncationERAttributeDTO attribute : entity.getAttributes()) {
                     setAttributeEn(attribute, wordsList);
                 }
             }
@@ -180,7 +182,7 @@ public class EREDTO {
             targetTypeFirst.ifPresent(chineseEnglishWords -> relation.setTargetTypeEn(chineseEnglishWords.getEnglishWord()));
 
             if (CollUtil.isNotEmpty(relation.getAttributes())){
-                for (ERAttributeDTO attribute : relation.getAttributes()) {
+                for (TruncationERAttributeDTO attribute : relation.getAttributes()) {
                     setAttributeEn(attribute, wordsList);
                 }
             }
@@ -188,7 +190,7 @@ public class EREDTO {
 
     }
 
-    private void setAttributeEn(ERAttributeDTO attribute,List<ChineseEnglishWords> wordsList) {
+    private void setAttributeEn(TruncationERAttributeDTO attribute, List<ChineseEnglishWords> wordsList) {
         if (null == attribute || CollUtil.isEmpty(wordsList)){
             return;
         }

src/main/java/com/supervision/pdfqaserver/dto/EntityExtractionDTO.java

@@ -31,7 +31,7 @@ public class EntityExtractionDTO {
      */
     private String name;
 
-    private List<ERAttributeDTO> attributes = new ArrayList<>();
+    private List<TruncationERAttributeDTO> attributes = new ArrayList<>();
 
     public EntityExtractionDTO() {
     }
@@ -43,7 +43,7 @@ public class EntityExtractionDTO {
         this.name = entityExtraction.getName();
     }
 
-    public EntityExtractionDTO(String truncationId, String entity, String name, List<ERAttributeDTO> attributes) {
+    public EntityExtractionDTO(String truncationId, String entity, String name, List<TruncationERAttributeDTO> attributes) {
         this.truncationId = truncationId;
         this.entity = entity;
         this.name = name;

src/main/java/com/supervision/pdfqaserver/dto/IntentDTO.java

@@ -0,0 +1,11 @@
+package com.supervision.pdfqaserver.dto;
+
+import lombok.Data;
+
+/**
+ * 意图实体
+ */
+@Data
+public class IntentDTO {
+
+}

src/main/java/com/supervision/pdfqaserver/dto/RelationExtractionDTO.java

@@ -46,7 +46,7 @@ public class RelationExtractionDTO {
 
     private String targetTypeEn;
 
-    private List<ERAttributeDTO> attributes;
+    private List<TruncationERAttributeDTO> attributes;
 
     public RelationExtractionDTO() {
     }
@@ -61,7 +61,7 @@ public class RelationExtractionDTO {
         this.targetType = relationExtraction.getTargetType();
     }
 
-    public RelationExtractionDTO(String truncationId, String source, String sourceType, String relation, String target, String targetType, List<ERAttributeDTO> attributes) {
+    public RelationExtractionDTO(String truncationId, String source, String sourceType, String relation, String target, String targetType, List<TruncationERAttributeDTO> attributes) {
         this.truncationId = truncationId;
         this.source = source;
         this.relation = relation;

src/main/java/com/supervision/pdfqaserver/dto/TruncateDTO.java

@@ -51,6 +51,23 @@ public class TruncateDTO {
         this.content = documentDTO.getContent();
     }
 
+    public TruncateDTO(DocumentDTO documentDTO,String content) {
+        this.documentId = Integer.parseInt(documentDTO.getId());
+        this.sectionId = documentDTO.getSectionId();
+        this.layoutType = documentDTO.getLayoutType().toString();
+        this.title = documentDTO.getTitle();
+        this.content = content;
+    }
+
+    public TruncateDTO(DocumentTruncation documentTruncation) {
+        this.id = documentTruncation.getId();
+        this.documentId = documentTruncation.getDocumentId();
+        this.sectionId = documentTruncation.getSectionId();
+        this.layoutType = documentTruncation.getLayoutType();
+        this.title = documentTruncation.getTitle();
+        this.content = documentTruncation.getContent();
+    }
+
     public DocumentTruncation toDocumentTruncation() {
         DocumentTruncation truncation = new DocumentTruncation();
         truncation.setDocumentId(this.documentId);

src/main/java/com/supervision/pdfqaserver/dto/TruncationERAttributeDTO.java

@@ -0,0 +1,68 @@
+package com.supervision.pdfqaserver.dto;
+
+import com.supervision.pdfqaserver.domain.TruncationErAttribute;
+import lombok.Data;
+
+/**
+ * 实体属性
+ */
+@Data
+public class TruncationERAttributeDTO {
+
+    private String id;
+
+    /**
+     * 片段实体属性表 既可以是truncation_entity_extraction表id也可以是truncation_relation_extraction表id
+     */
+    private String terId;
+
+    /**
+     * 类型 0：terId关联的id为实体 1：terId关联的id为关系
+     */
+    private String associationType;
+
+    /**
+     * 属性名
+     */
+    private String attribute;
+
+    private String attributeEn;
+
+    /**
+     * 属性值
+     */
+    private String value;
+
+    /**
+     * 数据类型 0：字符串 1：数字
+     */
+    private String dataType;
+
+    public TruncationERAttributeDTO() {
+    }
+
+    public TruncationERAttributeDTO(TruncationErAttribute truncationErAttribute) {
+        this.id = truncationErAttribute.getId();
+        this.terId = truncationErAttribute.getTerId();
+        this.associationType = truncationErAttribute.getAssociationType();
+        this.attribute = truncationErAttribute.getAttribute();
+        this.value = truncationErAttribute.getValue();
+        this.dataType = truncationErAttribute.getDataType();
+    }
+
+    public TruncationERAttributeDTO(String attribute, String value, String dataType) {
+        this.attribute = attribute;
+        this.value = value;
+        this.dataType = dataType;
+    }
+
+    public TruncationErAttribute toTruncationErAttribute() {
+        TruncationErAttribute truncationErAttribute = new TruncationErAttribute();
+        truncationErAttribute.setTerId(this.terId);
+        truncationErAttribute.setAssociationType(this.associationType);
+        truncationErAttribute.setAttribute(this.attribute);
+        truncationErAttribute.setValue(this.value);
+        truncationErAttribute.setDataType(this.dataType);
+        return truncationErAttribute;
+    }
+}

src/main/java/com/supervision/pdfqaserver/service/AiCallService.java

@@ -0,0 +1,10 @@
+package com.supervision.pdfqaserver.service;
+
+/**
+ * @description: AI调用服务
+ */
+public interface AiCallService {
+
+
+    String call(String prompt);
+}

src/main/java/com/supervision/pdfqaserver/service/DomainMetadataService.java

@@ -2,6 +2,8 @@ package com.supervision.pdfqaserver.service;
 
 import com.supervision.pdfqaserver.domain.DomainMetadata;
 import com.baomidou.mybatisplus.extension.service.IService;
+import com.supervision.pdfqaserver.dto.DomainMetadataDTO;
+import java.util.List;
 
 /**
 * @author Administrator
@@ -11,5 +13,29 @@ import com.baomidou.mybatisplus.extension.service.IService;
 public interface DomainMetadataService extends IService<DomainMetadata> {
 
 
-    void  saveIfNotExists(DomainMetadata metadata);
+    /**
+     * 由于领域元数据表中加入了行业分类ID，所以该方法已不再使用
+     * 建议使用 saveIfNotExists(DomainMetadata metadata,String domainCategoryId)
+     * @param metadata
+     */
+    @Deprecated(since = "v_0.0.2")
+    void saveIfNotExists(DomainMetadata metadata);
+
+
+    void saveIfNotExists(DomainMetadata metadata, String domainCategoryId);
+
+
+    void batchSaveOrUpdateMetadata(List<DomainMetadataDTO> metadatas,String intentionId,String domainCategoryId);
+    
+    void completeSave(DomainMetadataDTO domainMetadataDTO);
+
+    /**
+     * 根据主键查询数据
+     * @param sourceType 源类型
+     * @param targetType 目标类型
+     * @param relation 关系
+     * @param domainCategoryId 行业分类ID
+     * @return
+     */
+    DomainMetadata getByPrimaryKey(String sourceType, String targetType, String relation,String domainCategoryId);
 }

src/main/java/com/supervision/pdfqaserver/service/IntentionDomainMetadataService.java

@@ -3,6 +3,8 @@ package com.supervision.pdfqaserver.service;
 import com.supervision.pdfqaserver.domain.IntentionDomainMetadata;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.List;
+
 /**
 * @author Administrator
 * @description 针对表【intention_domain_metadata】的数据库操作Service
@@ -10,4 +12,8 @@ import com.baomidou.mybatisplus.extension.service.IService;
 */
 public interface IntentionDomainMetadataService extends IService<IntentionDomainMetadata> {
 
+
+    List<IntentionDomainMetadata> listByIntentionId(String intentionId);
+
+    void batchSaveIfAbsent(String intentionId, List<String> metadataIds);
 }

src/main/java/com/supervision/pdfqaserver/service/IntentionService.java

@@ -1,8 +1,11 @@
 package com.supervision.pdfqaserver.service;
 
+import cn.hutool.core.util.StrUtil;
 import com.supervision.pdfqaserver.domain.Intention;
 import com.baomidou.mybatisplus.extension.service.IService;
 
+import java.util.List;
+
 /**
 * @author Administrator
 * @description 针对表【intention】的数据库操作Service
@@ -10,4 +13,22 @@ import com.baomidou.mybatisplus.extension.service.IService;
 */
 public interface IntentionService extends IService<Intention> {
 
+
+    /**
+     * 保存意图
+     * @param intents 意图
+     * @param domainCategoryId 行业分类ID
+     *@param truncateId 分片id
+     * @return
+     */
+    List<Intention> batchSaveIfAbsent(List<String> intents, String domainCategoryId, String truncateId);
+
+    /**
+     * 查询意图
+     * digest 意图 + 行业分类ID domainCategoryId = 唯一
+     * @param digest 意图
+     * @param domainCategoryId 行业分类ID
+     * @return
+     */
+    Intention queryByDigestAndDomainCategoryId(String digest, String domainCategoryId);
 }

src/main/java/com/supervision/pdfqaserver/service/IntentionTruncationService.java

@@ -10,4 +10,6 @@ import com.baomidou.mybatisplus.extension.service.IService;
 */
 public interface IntentionTruncationService extends IService<IntentionTruncation> {
 
+
+    void saveIfAbsent(String intentionId, String truncateId);
 }

src/main/java/com/supervision/pdfqaserver/service/KnowledgeGraphService.java

@@ -16,6 +16,22 @@ public interface KnowledgeGraphService {
      */
     void generateGraph(String pdfId);
 
+    /**
+     * 元数据训练
+     * @param pdfId pdfId
+     */
+    void metaDataTrain(Integer pdfId);
+
+
+    /**
+     * 基于训练结果生成知识图谱
+     * @param pdfId pdfId
+     */
+    void generateGraphBaseTrain(Integer pdfId);
+
+
+    TripleConversionPipeline getTripleConversionPipeline(String contentType,String industry);
+
     void generateGraph(List<EREDTO> eredtoList);
 
     List<EREDTO> truncateERE(List<TruncateDTO> truncateDTOS);

src/main/java/com/supervision/pdfqaserver/service/PdfInfoService.java

@@ -19,4 +19,20 @@ public interface PdfInfoService extends IService<PdfInfo> {
     void pdfToGraphFail(Integer pdfId);
 
     List<PdfInfo> listNeedGenerateGraph(Integer limit);
+
+    PdfInfo getByPdfId(Integer pdfId);
+
+
+    void updateContentType(Integer pdfId, String contentType);
+
+
+    void updateCategory(Integer pdfId, String category);
+
+
+    void pdfTrainStart(Integer pdfId);
+
+    void pdfTrainComplete(Integer pdfId);
+
+    void pdfTrainFail(Integer pdfId);
+
 }

src/main/java/com/supervision/pdfqaserver/service/TripleConversionPipeline.java

@@ -1,10 +1,7 @@
 package com.supervision.pdfqaserver.service;
 
-import com.supervision.pdfqaserver.dto.EREDTO;
-import com.supervision.pdfqaserver.dto.DocumentDTO;
-import com.supervision.pdfqaserver.dto.TableTitleDTO;
-import com.supervision.pdfqaserver.dto.TruncateDTO;
-
+import com.supervision.pdfqaserver.constant.DocumentContentTypeEnum;
+import com.supervision.pdfqaserver.dto.*;
 import java.util.List;
 
 /**
@@ -12,6 +9,56 @@ import java.util.List;
  */
 public interface TripleConversionPipeline {
 
+
+    /**
+     * 识别出PDF文档类型
+     * @param pdfId 文档ID
+     * @return DocumentContentTypeEnum 文档类型
+     */
+    DocumentContentTypeEnum makeOutPdfContentType(Integer pdfId);
+
+
+    /**
+     * 识别出PDF文档行业
+     * @param pdfId 文档ID
+     * @return 行业
+     */
+    String makeOutPdfIndustry(Integer pdfId);
+
+
+    /**
+     * 识别出truncate的意图,训练过程中使用
+     * @param truncate 切分文档
+     * @return String
+     */
+    List<String> makeOutTruncationIntent(TruncateDTO truncate);
+
+    /**
+     * 识别出truncate的意图,非训练过程中使用
+     * @param truncate 切分文档
+     * @param intents 意图列表
+     * @return
+     */
+
+    List<IntentDTO> makeOutTruncationIntent(TruncateDTO truncate, List<IntentDTO> intents);
+
+
+    /**
+     * 识别出truncate的领域元数据
+     * @param truncate 切分文档
+     * @return DomainMetadataDTO
+     */
+    List<DomainMetadataDTO> makeOutDomainMetadata(TruncateDTO truncate,List<String> intents);
+
+
+    /**
+     * 实体关系抽取
+     * @param truncateDTO 切分文档
+     * @param intents 意图
+     * @return
+     */
+    EREDTO doEre(TruncateDTO truncateDTO,List<IntentDTO> intents);
+
     /**
      * 切分文档
      * @param documents 文档列表

src/main/java/com/supervision/pdfqaserver/service/impl/DomainMetadataServiceImpl.java

@@ -1,20 +1,36 @@
 package com.supervision.pdfqaserver.service.impl;
 
+import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.lang.Assert;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.supervision.pdfqaserver.domain.DomainMetadata;
+import com.supervision.pdfqaserver.dto.DomainMetadataDTO;
 import com.supervision.pdfqaserver.service.DomainMetadataService;
 import com.supervision.pdfqaserver.mapper.DomainMetadataMapper;
+import com.supervision.pdfqaserver.service.ErAttributeService;
+import com.supervision.pdfqaserver.service.IntentionDomainMetadataService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.util.List;
 
 /**
 * @author Administrator
 * @description 针对表【domain_metadata(领域元数据)】的数据库操作Service实现
 * @createDate 2025-04-27 11:45:24
 */
+@Slf4j
 @Service
+@RequiredArgsConstructor
 public class DomainMetadataServiceImpl extends ServiceImpl<DomainMetadataMapper, DomainMetadata>
     implements DomainMetadataService{
 
+
+    private final IntentionDomainMetadataService intentionDomainMetadataService;
+
+    private final ErAttributeService erAttributeService;
     @Override
     public void saveIfNotExists(DomainMetadata metadata) {
 
@@ -26,6 +42,60 @@ public class DomainMetadataServiceImpl extends ServiceImpl<DomainMetadataMapper,
             this.save(metadata);
         }
     }
+
+    @Override
+    public void saveIfNotExists(DomainMetadata metadata, String domainCategoryId) {
+        DomainMetadata data = this.getByPrimaryKey(metadata.getSourceType(), metadata.getTargetType(), metadata.getRelation(), domainCategoryId);
+        if (null != data){
+            metadata.setId(data.getId());
+        }else {
+            super.save(metadata);
+        }
+    }
+
+    @Override
+    @Transactional(rollbackFor = Exception.class)
+    public void batchSaveOrUpdateMetadata(List<DomainMetadataDTO> metadatas,String intentionId,String domainCategoryId) {
+        if (CollUtil.isEmpty(metadatas)){
+            return;
+        }
+        Assert.notEmpty(domainCategoryId, "行业分类ID不能为空");
+        Assert.notEmpty(intentionId, "意图ID不能为空");
+
+        for (DomainMetadataDTO metadata : metadatas) {
+            DomainMetadata data = this.getByPrimaryKey(metadata.getSourceType(), metadata.getTargetType(), metadata.getRelation(), domainCategoryId);
+            if (null != data){
+                metadata.setId(data.getId());
+            }else {
+                DomainMetadata domainMetadata = metadata.toDomainMetadata();
+                this.saveIfNotExists(domainMetadata, domainCategoryId);
+                metadata.setId(domainMetadata.getId());
+            }
+        }
+    }
+
+    @Override
+    @Transactional(rollbackFor = Exception.class)
+    public void completeSave(DomainMetadataDTO domainMetadataDTO) {
+      /*  if (CollUtil.isEmpty(domainMetadataDTOS)){
+            return;
+        }
+        for (DomainMetadataDTO domainMetadataDTO : domainMetadataDTOS) {
+            DomainMetadata domainMetadata = domainMetadataDTO.toDomainMetadata();
+            this.saveIfNotExists(domainMetadata);
+            // 保存属性信息
+        }*/
+    }
+
+    @Override
+    public DomainMetadata getByPrimaryKey(String sourceType, String targetType, String relation, String domainCategoryId) {
+        return this.lambdaQuery()
+                .eq(DomainMetadata::getSourceType, sourceType)
+                .eq(DomainMetadata::getTargetType, targetType)
+                .eq(DomainMetadata::getRelation, relation)
+                .eq(DomainMetadata::getDomainCategoryId, domainCategoryId)
+                .one();
+    }
 }
 
 

src/main/java/com/supervision/pdfqaserver/service/impl/IntentionDomainMetadataServiceImpl.java

@@ -1,10 +1,16 @@
 package com.supervision.pdfqaserver.service.impl;
 
+import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.lang.Assert;
+import cn.hutool.core.util.StrUtil;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.supervision.pdfqaserver.domain.IntentionDomainMetadata;
 import com.supervision.pdfqaserver.service.IntentionDomainMetadataService;
 import com.supervision.pdfqaserver.mapper.IntentionDomainMetadataMapper;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
 * @author Administrator
@@ -15,6 +21,35 @@ import org.springframework.stereotype.Service;
 public class IntentionDomainMetadataServiceImpl extends ServiceImpl<IntentionDomainMetadataMapper, IntentionDomainMetadata>
     implements IntentionDomainMetadataService{
 
+    @Override
+    public List<IntentionDomainMetadata> listByIntentionId(String intentionId) {
+        if (StrUtil.isEmpty(intentionId)){
+            return new ArrayList<>();
+        }
+        return this.lambdaQuery().eq(IntentionDomainMetadata::getIntentionId, intentionId).list();
+    }
+
+    @Override
+    @Transactional(rollbackFor = Exception.class)
+    public void batchSaveIfAbsent(String intentionId, List<String> metadataIds) {
+        if (CollUtil.isEmpty(metadataIds)){
+            return;
+        }
+        Assert.notEmpty(intentionId, "意图ID不能为空");
+
+        List<IntentionDomainMetadata> intentMetadataList = this.listByIntentionId(intentionId);
+        List<String> list = metadataIds.stream().filter(metadataId -> intentMetadataList.stream().noneMatch(item -> item.getDomainMetadataId().equals(metadataId))).toList();
+        if (CollUtil.isNotEmpty(list)){
+            List<IntentionDomainMetadata> doms = new ArrayList<>();
+            for (String metadataId : list) {
+                IntentionDomainMetadata idm = new IntentionDomainMetadata();
+                idm.setIntentionId(intentionId);
+                idm.setDomainMetadataId(metadataId);
+                doms.add(idm);
+            }
+            this.saveBatch(doms);
+       }
+    }
 }
 
 

src/main/java/com/supervision/pdfqaserver/service/impl/IntentionServiceImpl.java

@@ -1,20 +1,65 @@
 package com.supervision.pdfqaserver.service.impl;
 
+import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.lang.Assert;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.supervision.pdfqaserver.domain.Intention;
 import com.supervision.pdfqaserver.service.IntentionService;
 import com.supervision.pdfqaserver.mapper.IntentionMapper;
+import com.supervision.pdfqaserver.service.IntentionTruncationService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
 * @author Administrator
 * @description 针对表【intention】的数据库操作Service实现
 * @createDate 2025-05-14 15:23:54
 */
+@Slf4j
 @Service
+@RequiredArgsConstructor
 public class IntentionServiceImpl extends ServiceImpl<IntentionMapper, Intention>
     implements IntentionService{
 
+    private final IntentionTruncationService intentionTruncationService;
+    @Override
+    public List<Intention> batchSaveIfAbsent(List<String> intents, String domainCategoryId,String truncateId) {
+        Assert.notEmpty(domainCategoryId, "行业分类ID不能为空");
+        if (CollUtil.isEmpty(intents)){
+            return new ArrayList<>();
+        }
+        List<Intention> intentions = this.lambdaQuery().in(Intention::getDigest, intents)
+                .eq(Intention::getDomainCategoryId, domainCategoryId).list();
+
+        List<Intention> result = new ArrayList<>();
+        for (String intent : intents) {
+            Intention one = intentions.stream().filter(i -> i.getDigest().equals(intent)).findFirst().orElse(null);
+            if (null == one){
+                Intention intention = new Intention();
+                intention.setDigest(intent);
+                intention.setDomainCategoryId(domainCategoryId);
+                intention.setGenerationType("1");
+                this.save(intention);
+                result.add(intention);
+
+                // 保存关联关系
+                intentionTruncationService.saveIfAbsent(intention.getId(), truncateId);
+            }else {
+                result.add(one);
+                intentionTruncationService.saveIfAbsent(one.getId(), truncateId);
+            }
+        }
+        return result;
+    }
+
+    @Override
+    public Intention queryByDigestAndDomainCategoryId(String digest, String domainCategoryId) {
+        return this.lambdaQuery().eq(Intention::getDigest, digest)
+                .eq(Intention::getDomainCategoryId, domainCategoryId).one();
+    }
 }
 
 

src/main/java/com/supervision/pdfqaserver/service/impl/IntentionTruncationServiceImpl.java

@@ -1,5 +1,6 @@
 package com.supervision.pdfqaserver.service.impl;
 
+import cn.hutool.core.lang.Assert;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.supervision.pdfqaserver.domain.IntentionTruncation;
 import com.supervision.pdfqaserver.service.IntentionTruncationService;
@@ -15,6 +16,19 @@ import org.springframework.stereotype.Service;
 public class IntentionTruncationServiceImpl extends ServiceImpl<IntentionTruncationMapper, IntentionTruncation>
     implements IntentionTruncationService{
 
+    @Override
+    public void saveIfAbsent(String intentionId, String truncateId) {
+        Assert.notEmpty(intentionId, "意图ID不能为空");
+        Assert.notEmpty(truncateId, "分片ID不能为空");
+        IntentionTruncation intentionTruncation = this.lambdaQuery().eq(IntentionTruncation::getIntentionId, intentionId)
+                .eq(IntentionTruncation::getTruncateId, truncateId).one();
+        if (null == intentionTruncation){
+            IntentionTruncation truncation = new IntentionTruncation();
+            truncation.setIntentionId(intentionId);
+            truncation.setTruncateId(truncateId);
+            this.save(truncation);
+        }
+    }
 }
 
 

src/main/java/com/supervision/pdfqaserver/service/impl/KnowledgeGraphServiceImpl.java

@@ -2,14 +2,15 @@ package com.supervision.pdfqaserver.service.impl;
 
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.TimeInterval;
+import cn.hutool.core.lang.Assert;
 import cn.hutool.core.util.NumberUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONUtil;
+import com.supervision.pdfqaserver.constant.DocumentContentTypeEnum;
 import com.supervision.pdfqaserver.constant.DomainMetaGenerationEnum;
 import com.supervision.pdfqaserver.domain.*;
 import com.supervision.pdfqaserver.dto.*;
 import com.supervision.pdfqaserver.service.*;
-import com.supervision.pdfqaserver.thread.KnowledgeGraphGenerateTreadPool;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.aop.framework.AopContext;
@@ -46,6 +47,10 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
 
     private final ChinesEsToEnglishGenerator chinesEsToEnglishGenerator;
 
+    private final PdfInfoService pdfInfoService;
+
+    private final IntentionService intentionService;
+
     @Override
     public void generateGraph(String pdfId) {
 
@@ -76,6 +81,79 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
 
     }
 
+    @Override
+    public void metaDataTrain(Integer pdfId) {
+        Assert.notNull(pdfId, "pdfId不能为空");
+        PdfInfo pdfInfo = pdfInfoService.getByPdfId(pdfId);
+        Assert.notNull(pdfInfo, "pdfId:{}没有找到对应的pdf信息", pdfId);
+        if (null == pdfInfo.getTrainStatus()){
+            log.info("pdfId:{}没有找到对应的pdf训练状态,开始识别文档训练状态...", pdfId);
+            pdfInfoService.pdfToGraphStart(pdfId);
+            if (StrUtil.isEmpty(pdfInfo.getContentType())){
+                log.info("pdfId:{}没有找到对应的pdf内容类型,开始识别文档内容类型...", pdfId);
+                DocumentContentTypeEnum documentContentTypeEnum = tripleConversionPipeline.makeOutPdfContentType(pdfId);
+                log.info("pdfId:{}识别文档内容类型完成,内容类型:{}", pdfId, documentContentTypeEnum.getType());
+                pdfInfo.setContentType(documentContentTypeEnum.getType());
+                pdfInfoService.updateContentType(pdfId, documentContentTypeEnum.getType());
+            }
+            if (StrUtil.isEmpty(pdfInfo.getDomainCategoryId())){
+                log.info("pdfId:{}没有找到对应的pdf行业,开始识别文档行业...", pdfId);
+                String industry = tripleConversionPipeline.makeOutPdfIndustry(pdfId);
+                log.info("pdfId:{}识别文档行业完成,行业:{}", pdfId, industry);
+                pdfInfo.setDomainCategoryId(industry);
+                pdfInfoService.updateCategory(pdfId, industry);
+            }
+        }
+        TripleConversionPipeline tripleConversionPipeline = this.getTripleConversionPipeline(pdfInfo.getContentType(), pdfInfo.getDomainCategoryId());
+
+        List<PdfAnalysisOutput> pdfAnalysisOutputs = pdfAnalysisOutputService.queryByPdfId(pdfId);
+        List<String> documentIds = pdfAnalysisOutputs.stream().map(p->String.valueOf(p.getId())).collect(Collectors.toList());
+        List<DocumentTruncation> documentTruncations = documentTruncationService.queryByDocumentIds(documentIds);
+        List<TruncateDTO> truncateDTOS = new ArrayList<>();
+        if (CollUtil.isNotEmpty(documentTruncations)){
+            log.info("没有找到文档切分数据,pdfId:{},不用重置数据...", pdfId);
+            truncateDTOS = documentTruncations.stream().map(TruncateDTO::new).collect(Collectors.toList());
+        }
+        if (CollUtil.isEmpty(documentTruncations)){
+            log.info("开始切割文档切片,pdfId:{}", pdfId);
+            List<DocumentDTO> documentDTOList = pdfAnalysisOutputs.stream().map(DocumentDTO::new).collect(Collectors.toList());
+            truncateDTOS = tripleConversionPipeline.sliceDocuments(documentDTOList);
+            log.info("切割文档切片完成,切片个数:{}", truncateDTOS.size());
+            // 保存分片信息
+            documentTruncationService.batchSave(truncateDTOS);
+        }
+        for (TruncateDTO truncateDTO : truncateDTOS) {
+            List<String> intents = tripleConversionPipeline.makeOutTruncationIntent(truncateDTO);
+            List<DomainMetadataDTO> domainMetadataDTOS = tripleConversionPipeline.makeOutDomainMetadata(truncateDTO, intents);
+            // 保存意图数据
+            List<Intention> intentions = intentionService.batchSaveIfAbsent(intents, pdfInfo.getDomainCategoryId(), pdfId.toString());
+
+            for (Intention intention : intentions) {
+                List<DomainMetadataDTO> metadataDTOS = domainMetadataDTOS.stream()
+                        .filter(d -> StrUtil.equals(d.getIntentDigest(), intention.getDigest())).toList();
+                domainMetadataService.batchSaveOrUpdateMetadata(metadataDTOS,intention.getId(), pdfInfo.getDomainCategoryId());
+            }
+
+
+        }
+
+
+
+
+    }
+
+    @Override
+    public void generateGraphBaseTrain(Integer pdfId) {
+
+    }
+
+    @Override
+    public TripleConversionPipeline getTripleConversionPipeline(String contentType, String industry) {
+        // 内容类型决定了文本片段的切分方式,行业类别决定了文本片段的意图
+        // 内容类型和行业类型确定tripleConversionPipeline的具体实现方式，现在默认是pdf类型
+        return this.tripleConversionPipeline;
+    }
+
     @Override
     public void generateGraph(List<EREDTO> eredtoList) {
         log.info("开始合并实体关系抽取结果...");
@@ -108,7 +186,7 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
                 for (EntityExtractionDTO entityDTO : entities) {
                     saveWordsIfNecessary(entityDTO.getEntity(), allWords);
                     if (CollUtil.isNotEmpty(entityDTO.getAttributes())){
-                        for (ERAttributeDTO attribute : entityDTO.getAttributes()) {
+                        for (TruncationERAttributeDTO attribute : entityDTO.getAttributes()) {
                             saveWordsIfNecessary(attribute.getAttribute(), allWords);
                         }
                     }
@@ -119,7 +197,7 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
                 for (RelationExtractionDTO relationDTO : relations) {
                     saveWordsIfNecessary(relationDTO.getRelation(), allWords);
                     if (CollUtil.isNotEmpty(relationDTO.getAttributes())){
-                        for (ERAttributeDTO attribute : relationDTO.getAttributes()) {
+                        for (TruncationERAttributeDTO attribute : relationDTO.getAttributes()) {
                             saveWordsIfNecessary(attribute.getAttribute(), allWords);
                         }
                     }
@@ -294,8 +372,8 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
         for (TruncationEntityExtraction entityExtraction : truncationEntityExtractions) {
             EREDTO eredto = new EREDTO();
             EntityExtractionDTO extractionDTO = new EntityExtractionDTO(entityExtraction);
-            List<ERAttributeDTO> attributes = truncationErAttributes.stream()
-                    .filter(t -> StrUtil.equals(entityExtraction.getId(), t.getTerId())).map(ERAttributeDTO::new).collect(Collectors.toList());
+            List<TruncationERAttributeDTO> attributes = truncationErAttributes.stream()
+                    .filter(t -> StrUtil.equals(entityExtraction.getId(), t.getTerId())).map(TruncationERAttributeDTO::new).collect(Collectors.toList());
             extractionDTO.setAttributes(attributes);
             eredto.getEntities().add(extractionDTO);
             eres.add(eredto);
@@ -303,8 +381,8 @@ public class KnowledgeGraphServiceImpl implements KnowledgeGraphService {
         for (TruncationRelationExtraction relationExtraction : truncationRelationExtractions) {
             EREDTO eredto = new EREDTO();
             RelationExtractionDTO extractionDTO = new RelationExtractionDTO(relationExtraction);
-            List<ERAttributeDTO> attributes = truncationErAttributes.stream()
-                    .filter(t -> StrUtil.equals(relationExtraction.getId(), t.getTerId())).map(ERAttributeDTO::new).collect(Collectors.toList());
+            List<TruncationERAttributeDTO> attributes = truncationErAttributes.stream()
+                    .filter(t -> StrUtil.equals(relationExtraction.getId(), t.getTerId())).map(TruncationERAttributeDTO::new).collect(Collectors.toList());
             extractionDTO.setAttributes(attributes);
             eredto.getRelations().add(extractionDTO);
             eres.add(eredto);

src/main/java/com/supervision/pdfqaserver/service/impl/OllamaCallServiceImpl.java

@@ -0,0 +1,19 @@
+package com.supervision.pdfqaserver.service.impl;
+
+import com.supervision.pdfqaserver.service.AiCallService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.ai.ollama.OllamaChatModel;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class OllamaCallServiceImpl implements AiCallService {
+
+    private final OllamaChatModel ollamaChatModel;
+    @Override
+    public String call(String prompt) {
+        return null;
+    }
+}

src/main/java/com/supervision/pdfqaserver/service/impl/PdfInfoServiceImpl.java

@@ -47,6 +47,52 @@ public class PdfInfoServiceImpl extends ServiceImpl<PdfInfoMapper, PdfInfo>
                 .last("limit " + limit)
                 .list();
     }
+
+    @Override
+    public PdfInfo getByPdfId(Integer pdfId) {
+        return this.getById(pdfId);
+    }
+
+    @Override
+    public void updateContentType(Integer pdfId, String contentType) {
+        this.lambdaUpdate().eq(PdfInfo::getId, pdfId)
+                .set(PdfInfo::getContentType, contentType)
+                .update();
+    }
+
+    @Override
+    public void updateCategory(Integer pdfId, String category) {
+        this.lambdaUpdate().eq(PdfInfo::getId, pdfId)
+                .set(PdfInfo::getDomainCategoryId, category)
+                .update();
+    }
+
+    @Override
+    public void pdfTrainStart(Integer pdfId) {
+        this.lambdaUpdate().eq(PdfInfo::getId, pdfId)
+                .set(PdfInfo::getTrainStatus, 0)
+                .set(PdfInfo::getTrainStartTime, LocalDateTime.now())
+                .update();
+    }
+
+    @Override
+    public void pdfTrainComplete(Integer pdfId) {
+
+        this.lambdaUpdate().eq(PdfInfo::getId, pdfId)
+                .set(PdfInfo::getTrainStatus, 1)
+                .set(PdfInfo::getTrainEndTime, LocalDateTime.now())
+                .update();
+    }
+
+    @Override
+    public void pdfTrainFail(Integer pdfId) {
+        this.lambdaUpdate().eq(PdfInfo::getId, pdfId)
+                .set(PdfInfo::getTrainStatus, 2)
+                .set(PdfInfo::getTrainEndTime, LocalDateTime.now())
+                .update();
+    }
+
+
 }
 
 

src/main/java/com/supervision/pdfqaserver/service/impl/TripleConversionPipelineImpl.java

@@ -6,6 +6,7 @@ import cn.hutool.core.util.BooleanUtil;
 import cn.hutool.core.util.RandomUtil;
 import cn.hutool.core.util.StrUtil;
 import com.supervision.pdfqaserver.cache.PromptCache;
+import com.supervision.pdfqaserver.constant.DocumentContentTypeEnum;
 import com.supervision.pdfqaserver.constant.LayoutTypeEnum;
 import com.supervision.pdfqaserver.dto.*;
 import com.supervision.pdfqaserver.service.TripleConversionPipeline;
@@ -27,8 +28,50 @@ public class TripleConversionPipelineImpl implements TripleConversionPipeline {
 
     private final OllamaChatModel ollamaChatModel;
 
+    @Override
+    public DocumentContentTypeEnum makeOutPdfContentType(Integer pdfId) {
+        return null;
+    }
+
+    @Override
+    public String makeOutPdfIndustry(Integer pdfId) {
+        return null;
+    }
+
+    @Override
+    public List<String> makeOutTruncationIntent(TruncateDTO truncate) {
+        return null;
+    }
+
+    @Override
+    public List<IntentDTO> makeOutTruncationIntent(TruncateDTO truncate, List<IntentDTO> intents) {
+        return null;
+    }
+
+    @Override
+    public List<DomainMetadataDTO> makeOutDomainMetadata(TruncateDTO truncate,List<String> intents) {
+        return null;
+    }
+
+    @Override
+    public EREDTO doEre(TruncateDTO truncateDTO, List<IntentDTO> intents) {
+        return null;
+    }
+
+    /**
+     * 切分文档
+     * 切分规则：
+     *       文本类型: 以单句为最小单元，最大字数现在这1000字以内。单句超过1000字取完成的单句。
+     *      表格类型: 以4行数据为最小单元。
+     * @param documents 文档列表
+     * @return
+     */
     @Override
     public List<TruncateDTO> sliceDocuments(List<DocumentDTO> documents) {
+
+        int maxTextLength = 1000;
+        int minTextLength = 800;
+        int INITIAL_BUFFER_SIZE = 1500;
         // 对pdfAnalysisOutputs进行排序
         List<DocumentDTO> documentDTOList = documents.stream().sorted(
                 // 先对pageNo进行排序再对layoutOrder进行排序
@@ -45,6 +88,7 @@ public class TripleConversionPipelineImpl implements TripleConversionPipeline {
         // 创建管道
         StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
         List<TruncateDTO> truncateDTOS = new ArrayList<>();
+        StringBuilder truncateTextBuild = new StringBuilder(1500);
         for (DocumentDTO documentDTO : documentDTOList) {
             String content = documentDTO.getContent();
             if (StrUtil.isEmpty(content)){
@@ -58,9 +102,30 @@ public class TripleConversionPipelineImpl implements TripleConversionPipeline {
                 pipeline.annotate(document);
                 // 获取句子
                 for (CoreSentence sentence : document.sentences()) {
-                    TruncateDTO truncateDTO = new TruncateDTO(documentDTO);
-                    truncateDTO.setContent(sentence.text());
-                    truncateDTOS.add(truncateDTO);
+                    if (StrUtil.isEmpty(sentence.text())) {
+                        continue;
+                    }
+                    if (sentence.text().length() >= maxTextLength) {
+                        if (truncateTextBuild.length() >= minTextLength) {
+                            // 提交缓存内容
+                            truncateDTOS.add(new TruncateDTO(documentDTO, truncateTextBuild.toString()));
+                            truncateTextBuild = new StringBuilder(INITIAL_BUFFER_SIZE);
+                        }
+                        // 提交超长句子
+                        truncateDTOS.add(new TruncateDTO(documentDTO, sentence.text()));
+                    } else {
+                        if (truncateTextBuild.length() + sentence.text().length() >= minTextLength) {
+                            truncateTextBuild.append(sentence.text());
+                            truncateDTOS.add(new TruncateDTO(documentDTO, truncateTextBuild.toString()));
+                            truncateTextBuild = new StringBuilder(INITIAL_BUFFER_SIZE);
+                        } else {
+                            truncateTextBuild.append(sentence.text());
+                        }
+                    }
+                }
+                // 处理剩余内容
+                if (!truncateTextBuild.isEmpty()) {
+                    truncateDTOS.add(new TruncateDTO(documentDTO, truncateTextBuild.toString()));
                 }
             } else if (LayoutTypeEnum.TABLE.getCode() == layoutType) {
                 // 如果是表格类型的布局，进行切分
@@ -280,11 +345,11 @@ public class TripleConversionPipelineImpl implements TripleConversionPipeline {
                 return;
             }
             // 合并属性
-            List<ERAttributeDTO> cachedAttributes = cachedRelation.getAttributes();
+            List<TruncationERAttributeDTO> cachedAttributes = cachedRelation.getAttributes();
             if (null == cachedAttributes){
                 cachedAttributes = new ArrayList<>();
             }
-            for (ERAttributeDTO attribute : relation.getAttributes()) {
+            for (TruncationERAttributeDTO attribute : relation.getAttributes()) {
                 String attributeKey = attribute.getAttribute();
                 String attributeValue = attribute.getValue();
                 if (StrUtil.isEmpty(attributeKey) || StrUtil.isEmpty(attributeValue)){
@@ -307,12 +372,12 @@ public class TripleConversionPipelineImpl implements TripleConversionPipeline {
                 return;
             }
             // 合并属性
-            List<ERAttributeDTO> cachedAttributes = cachedEntity.getAttributes();
+            List<TruncationERAttributeDTO> cachedAttributes = cachedEntity.getAttributes();
             if (null == cachedAttributes){
                 cachedAttributes = new ArrayList<>();
                 cachedEntity.setAttributes(cachedAttributes);
             }
-            for (ERAttributeDTO attribute : entity.getAttributes()) {
+            for (TruncationERAttributeDTO attribute : entity.getAttributes()) {
                 String attributeKey = attribute.getAttribute();
                 String attributeValue = attribute.getValue();
                 if (StrUtil.isEmpty(attributeKey) || StrUtil.isEmpty(attributeValue)){

src/main/java/com/supervision/pdfqaserver/service/impl/TripleToCypherExecutorImpl.java

@@ -5,7 +5,7 @@ import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONUtil;
 import com.supervision.pdfqaserver.cache.PromptCache;
 import com.supervision.pdfqaserver.dao.Neo4jRepository;
-import com.supervision.pdfqaserver.dto.ERAttributeDTO;
+import com.supervision.pdfqaserver.dto.TruncationERAttributeDTO;
 import com.supervision.pdfqaserver.dto.EREDTO;
 import com.supervision.pdfqaserver.dto.EntityExtractionDTO;
 import com.supervision.pdfqaserver.dto.RelationExtractionDTO;
@@ -60,7 +60,7 @@ public class TripleToCypherExecutorImpl implements TripleToCypherExecutor {
                     continue;
                 }
                 Map<String, Object> attributes = entity.getAttributes().stream().collect(Collectors.toMap(
-                        ERAttributeDTO::getAttributeEn, ERAttributeDTO::getValue
+                        TruncationERAttributeDTO::getAttributeEn, TruncationERAttributeDTO::getValue
                 ));
                 attributes.put("truncationId", entity.getTruncationId());
                 attributes.put("name", entity.getName());
@@ -85,7 +85,7 @@ public class TripleToCypherExecutorImpl implements TripleToCypherExecutor {
                     continue;
                 }
                 Map<String, Object> attributes = relation.getAttributes().stream().collect(Collectors.toMap(
-                        ERAttributeDTO::getAttributeEn, ERAttributeDTO::getValue
+                        TruncationERAttributeDTO::getAttributeEn, TruncationERAttributeDTO::getValue
                 ));
                 attributes.put("sourceType", relation.getSourceType());
                 attributes.put("truncationId", relation.getTruncationId());

src/main/java/com/supervision/pdfqaserver/service/impl/TruncationEntityExtractionServiceImpl.java

@@ -5,7 +5,7 @@ import cn.hutool.core.util.StrUtil;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.supervision.pdfqaserver.domain.TruncationEntityExtraction;
 import com.supervision.pdfqaserver.domain.TruncationErAttribute;
-import com.supervision.pdfqaserver.dto.ERAttributeDTO;
+import com.supervision.pdfqaserver.dto.TruncationERAttributeDTO;
 import com.supervision.pdfqaserver.dto.EntityExtractionDTO;
 import com.supervision.pdfqaserver.service.TruncationEntityExtractionService;
 import com.supervision.pdfqaserver.mapper.TruncationEntityExtractionMapper;
@@ -39,11 +39,11 @@ public class TruncationEntityExtractionServiceImpl extends ServiceImpl<Truncatio
             tee.setEntity(entity.getEntity());
             tee.setName(entity.getName());
             this.save(tee);
-            List<ERAttributeDTO> attributes = entity.getAttributes();
+            List<TruncationERAttributeDTO> attributes = entity.getAttributes();
             if (CollUtil.isEmpty(attributes)){
                 continue;
             }
-            for (ERAttributeDTO attribute : attributes) {
+            for (TruncationERAttributeDTO attribute : attributes) {
                 attribute.setTerId(tee.getId());
                 TruncationErAttribute era = attribute.toTruncationErAttribute();
                 truncationErAttributeService.save(era);

src/main/java/com/supervision/pdfqaserver/service/impl/TruncationRelationExtractionServiceImpl.java

@@ -5,7 +5,7 @@ import cn.hutool.core.util.StrUtil;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.supervision.pdfqaserver.domain.TruncationErAttribute;
 import com.supervision.pdfqaserver.domain.TruncationRelationExtraction;
-import com.supervision.pdfqaserver.dto.ERAttributeDTO;
+import com.supervision.pdfqaserver.dto.TruncationERAttributeDTO;
 import com.supervision.pdfqaserver.dto.RelationExtractionDTO;
 import com.supervision.pdfqaserver.service.TruncationErAttributeService;
 import com.supervision.pdfqaserver.service.TruncationRelationExtractionService;
@@ -40,7 +40,7 @@ public class TruncationRelationExtractionServiceImpl extends ServiceImpl<Truncat
             if (CollUtil.isEmpty(relation.getAttributes())){
                 continue;
             }
-            for (ERAttributeDTO attribute : relation.getAttributes()) {
+            for (TruncationERAttributeDTO attribute : relation.getAttributes()) {
                 TruncationErAttribute era = attribute.toTruncationErAttribute();
                 era.setTerId(re.getId());
                 truncationErAttributeService.save(era);

src/main/resources/mapper/ErAttributeMapper.xml

@@ -11,7 +11,6 @@
             <result property="attrName" column="attr_name" jdbcType="VARCHAR"/>
             <result property="attrValueType" column="attr_value_type" jdbcType="VARCHAR"/>
             <result property="erType" column="er_type" jdbcType="VARCHAR"/>
-            <result property="parentId" column="parent_id" jdbcType="VARCHAR"/>
             <result property="createTime" column="create_time" jdbcType="TIMESTAMP"/>
             <result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
     </resultMap>
@@ -19,6 +18,6 @@
     <sql id="Base_Column_List">
         id,domain_metadata_id,er_name,
         attr_name,attr_value_type,er_type,
-        parent_id,create_time,update_time
+        create_time,update_time
     </sql>
 </mapper>

src/main/resources/mapper/IntentionMapper.xml

@@ -9,12 +9,13 @@
             <result property="digest" column="digest" jdbcType="VARCHAR"/>
             <result property="desc" column="desc" jdbcType="VARCHAR"/>
             <result property="domainCategoryId" column="domain_category_id" jdbcType="VARCHAR"/>
+            <result property="generationType" column="generation_type" jdbcType="VARCHAR"/>
             <result property="createTime" column="create_time" jdbcType="TIMESTAMP"/>
             <result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
     </resultMap>
 
     <sql id="Base_Column_List">
-        id,digest,desc,
+        id,digest,desc,generation_type,
         domain_category_id,create_time,update_time
     </sql>
 </mapper>