package com.supervision.demo; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.date.DateTime; import cn.hutool.core.date.DateUnit; import cn.hutool.core.date.DateUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import cn.hutool.poi.excel.ExcelUtil; import cn.hutool.poi.excel.ExcelWriter; import com.supervision.common.domain.R; import com.supervision.neo4j.controller.Neo4jController; import com.supervision.neo4j.domain.CaseNode; import com.supervision.demo.controller.ExampleChatController; import com.supervision.police.domain.ModelRecordType; import com.supervision.police.domain.NotePrompt; import com.supervision.police.domain.NoteRecordSplit; import com.supervision.police.dto.NotePromptExtractAttributesDto; import com.supervision.police.dto.RetrieveReqDTO; import com.supervision.police.dto.RetrieveResDTO; import com.supervision.police.service.*; import com.supervision.thread.RecordSplitClassifyTask; import lombok.AllArgsConstructor; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; import org.springframework.ai.chat.ChatResponse; import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.ollama.OllamaChatClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.util.StopWatch; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @Slf4j @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) public class FuHsiApplicationTests { @Autowired private OllamaChatClient ollamaChatClient; @Autowired private ExampleChatController exampleChatController; @Autowired private Neo4jController neo4jController; @Test public void contextLoads() { } @Test public void test() { // exampleChatController.test("1803663875373694977", "你现在是一个笔录分析人员,请用四个字描述一下下述内容属于哪种类型的对话?"); } @Test public void savePersion() { CaseNode caseNode = new CaseNode(); caseNode.setName("自然人"); R save = neo4jController.save(caseNode); System.out.printf(save.toString()); } private final OllamaChatClient chatClient; @Autowired public FuHsiApplicationTests(OllamaChatClient chatClient) { this.chatClient = chatClient; } @Autowired private NoteRecordSplitService noteRecordSplitService; @Autowired private ModelRecordTypeService modelRecordTypeService; @Test public void classificationTest() { String recordId = "1824329325387304962"; final String NEW_TEMPLATE = """ 分类任务: 将对话笔录文本进行分类。 目标: 将给定的对话分配到预定义的类别中 预定义类别为:{typeContext}。 说明: 提供一段对话笔记录文本,分类器应该识别出对话的主题,并将其归类到上述类别中的一个或多个。 --- 示例输入: 办案警官问:你和上述的这些受害人签订协议之后是否实际履行合同? 行为人XXX回答:我和他们签订合同就是为了骗他们相信我,我就是伪造的一些假合同,等我把他们的钱骗到手之后我就不会履行合同。 预期输出: {"result":[{"type":"合同和协议","explain":"行为人XXX提到签订合同"},{"type":"虚假信息和伪造","explain":"行为人XXX提到合同是假合同"}]} --- 任务要求: 1. 分类器应当准确地识别对话的主题,分类来自于预定义的类别。 2. 分类器应该实事求是按照对话进行分类,不要有过多的推测。 2. 如果一段对话笔记录包含多个主题,请选择最相关的类别,最多可选择三个分类。 3. 如果不涉及任何分类则回复{"result":[]} --- 以下为问答对内容: {question} {answer} --- 返回格式为json,字段名要严格一致:{"result":[{"type":"分类1","explain":"分类原因"},{"type":"分类2","explain":"分类原因"}]} """; final String TYPE_CONTEXT_TEMPLATE = "{分类type:{type},区别点(分类释义):{typeExt}}"; List allTypeList = modelRecordTypeService.lambdaQuery().list(); String type = ""; // 根据recordId查询所有的分割后的笔录 List noteRecordSplitList = noteRecordSplitService.lambdaQuery().eq(NoteRecordSplit::getNoteRecordId, recordId).list(); for (NoteRecordSplit noteRecordSplit : noteRecordSplitList) { try { StopWatch stopWatch = new StopWatch(); // 首先拼接分类模板 List typeContextList = new ArrayList<>(); for (ModelRecordType modelRecordType : allTypeList) { String format = StrUtil.format(TYPE_CONTEXT_TEMPLATE, Map.of("type", modelRecordType.getRecordType(), "typeExt", modelRecordType.getRecordTypeExt())); typeContextList.add(format); } // 开始对笔录进行分类 Map paramMap = new HashMap<>(); paramMap.put("typeContext", CollUtil.join(typeContextList, ";")); paramMap.put("question", noteRecordSplit.getQuestion()); paramMap.put("answer", noteRecordSplit.getAnswer()); Prompt prompt = new Prompt(new UserMessage(StrUtil.format(NEW_TEMPLATE, paramMap))); stopWatch.start(); log.info("开始分析:"); ChatResponse call = chatClient.call(prompt); stopWatch.stop(); log.info("耗时:{}", stopWatch.getTotalTimeSeconds()); String content = call.getResult().getOutput().getContent(); log.info("问:{}, 答:{}", noteRecordSplit.getQuestion(), noteRecordSplit.getAnswer()); log.info("分析的结果是:{}", content); RecordSplitClassifyTask.TypeResultDTO result = JSONUtil.toBean(content, RecordSplitClassifyTask.TypeResultDTO.class); List typeList = result.getResult(); if (CollUtil.isNotEmpty(typeList)) { // 将type进行拼接,并以分号进行分割 type = CollUtil.join(typeList.stream().map(RecordSplitClassifyTask.TypeNodeDTO::getType).collect(Collectors.toSet()), ";"); } else { // 如果没有提取到,就是无 type = "无"; } } catch (Exception e) { log.error("分类任务执行失败:{}", e.getMessage(), e); type = "无"; } log.info("最后结果 question:{},answer:{},分析的结果是:{}", noteRecordSplit.getQuestion(),noteRecordSplit.getAnswer(),type); } } @Autowired private RecordSplitClassifyService recordSplitClassifyService; @Test public void classificationTest2() { List typeList = modelRecordTypeService.lambdaQuery().list(); List noteRecordSplits = noteRecordSplitService.lambdaQuery().eq(NoteRecordSplit::getId, "1824729361214418946").list(); recordSplitClassifyService.classify(typeList,noteRecordSplits); log.info("分类结果:{}",noteRecordSplits); } @Autowired private OCRService ocrService; @Test public void retrieveTest() { String question = "银川市公安局金凤区分局\\r\\n拘留通知书\\r\\n(副本)\\r\\n银金公(经侦)拘通字[2024]10017号\\r\\n梁玉峰家属\\r\\n根据《中华人民共和国刑事诉讼法》第八十二条第(一)项之规定\\r\\n罪\\r\\n将涉嫌合同诈骗\\r\\n我局已于2024年01月16日10时\\r\\n银川市看守所\\r\\n的\\r\\n梁玉峰\\r\\n刑事拘留,现羁押在\\r\\n市\\r\\n二0二\\r\\n年名\\r\\n本通知书已收到。\\r\\n年月\\r\\n日时\\r\\n被拘留人家属:\\r\\n如未在拘留后24小时内通知被拘留人家属,注明原因:办集民警\\r\\n当天拔打梁天峰亲属果无辉毛话,对方电生无法接通\\r\\n办案人:\\r\\n2024年1月16日1时\\r\\n此联附卷"; RetrieveResDTO retrieve = ocrService.retrieve(new RetrieveReqDTO(question)); System.out.println(retrieve); } @Autowired private NotePromptService notePromptService; @Test public void promptTest() throws InterruptedException { List listPromptBySplitId = notePromptService.listPromptBySplitId("1825358898516275202"); ArrayList notePrompts = new ArrayList<>(); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); notePrompts.addAll(listPromptBySplitId); NoteRecordSplit noteRecordSplit = new NoteRecordSplit(); /*noteRecordSplit.setQuestion("办案警官问:你让马旭东帮你买车你们是如何协商的?"); noteRecordSplit.setAnswer("裴金禄回答:我和马旭东认识十年时间了,我们关系比较好。我找到马旭东告诉他我有车要出售,这样要求把帮忙出售,我不出面。出售完毕之后我再给马旭东支付一些好处费用");*/ noteRecordSplit.setQuestion("办案警官问:你和马旭东是如何协商好处费的?"); noteRecordSplit.setAnswer("裴金禄回答:我没有协商过,我都是随心意给马旭东给的,"); String mainActor = "裴金禄"; List tripleRecords = new ArrayList<>(); AtomicLong atomicLong = new AtomicLong(); CountDownLatch countDownLatch = new CountDownLatch(notePrompts.size()); ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 10000, 100L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); DateTime start = DateUtil.date(); for (NotePrompt notePrompt : notePrompts) { executor.submit(()->{ TripleRecord tripleRecord = chat4Triple(notePrompt, noteRecordSplit, mainActor,null); atomicLong.getAndAdd(tripleRecord.getTime()); countDownLatch.countDown(); tripleRecords.add(tripleRecord); }); } countDownLatch.await(); long between = DateUtil.between(start, DateUtil.date(), DateUnit.SECOND); log.info("实际总耗时:" + between); ExcelWriter writer = ExcelUtil.getWriter("F:\\tmp\\1\\ollama-模型优化\\50服务器\\prompt.xlsx","速度对比-并发1-qwen2.5-32b"); writer.write(tripleRecords); writer.close(); log.info("所有线程总耗时:" + atomicLong.get()); } private TripleRecord chat4Triple(NotePrompt prompt, NoteRecordSplit noteRecordSplit,String mainActor,String type) { StopWatch stopWatch = new StopWatch(); // 分析三元组 stopWatch.start(); HashMap paramMap = new HashMap<>(); paramMap.put("headEntityType", prompt.getStartEntityType()); paramMap.put("relation", prompt.getRelType()); paramMap.put("tailEntityType", prompt.getEndEntityType()); paramMap.put("question", noteRecordSplit.getQuestion()); paramMap.put("answer", noteRecordSplit.getAnswer()); //log.info("开始尝试提取三元组:{}-{}-{},mainActor:{}", prompt.getStartEntityType(), prompt.getRelType(), prompt.getEndEntityType(), mainActor == null ? "" : mainActor); if (mainActor != null && "行为人".equals(prompt.getStartEntityType())) { paramMap.put("requirement", "当前案件的行为人是" + mainActor + ",只尝试提取" + mainActor + "为头结点的三元组。"); } else { paramMap.put("requirement", ""); } String format = StrUtil.format(prompt.getPrompt(), paramMap); //log.info("提示词内容:{}", format); log.info("开始执行:chatClient.call 开始"); ChatResponse call = chatClient.call(new Prompt(new UserMessage(format))); log.info("开始执行:chatClient.call 结束"); stopWatch.stop(); String content = call.getResult().getOutput().getContent(); //log.info("问题:{}耗时:{},三元组提取结果是:{}", noteRecordSplit.getQuestion(), stopWatch.getTotalTimeSeconds(), content); return new TripleRecord(noteRecordSplit.getQuestion(),noteRecordSplit.getAnswer(),type, prompt.getId(),prompt.getPrompt(),format, content, stopWatch.getTotalTimeMillis()); } @Autowired private CaseEvidenceService caseEvidenceService; @Test public void ocrAndExtractTest() { // EvidenceDirectoryDTO directoryDTO = new EvidenceDirectoryDTO(); // directoryDTO.setId("1845745238182703105"); // directoryDTO.setDirectoryName("书证"); // EvidenceDirectoryDTO directoryDTO1 = new EvidenceDirectoryDTO(); // directoryDTO1.setId("1845745256180461570"); // directoryDTO1.setDirectoryName("合同协议"); // directoryDTO1.setParentId("1845745238182703105"); // directoryDTO1.setFileIdList(List.of("1823958525387853825","1831232334119645185")); // directoryDTO.setChild(List.of(directoryDTO1)); // // EvidenceDirectoryDTO directoryDTO2 = new EvidenceDirectoryDTO(); // directoryDTO2.setId("2222"); // directoryDTO2.setDirectoryName("车辆合同协议"); // directoryDTO2.setParentId("1845745256180461570"); // directoryDTO2.setFileIdList(List.of("1833015941205143554","1833016215680397313","1833016621840019457")); // directoryDTO1.setChild(List.of(directoryDTO2)); // caseEvidenceService.ocrAndExtract("1823955210189000706", List.of(directoryDTO)); } @Test public void initCaseDirectory() { caseEvidenceService.initCaseEvidenceDirectory("1846389691251220482", "1"); } @Data @AllArgsConstructor class TripleRecord{ private String question; private String answer; private String type; private String templateId; private String template; private String prompt; private String result; private long time; } @Test void refreshCaseEvidenceTest() { caseEvidenceService.refreshCaseEvidence(); System.out.println("执行完成"); } @Autowired private CaseEvidencePropertyService caseEvidencePropertyService; @Test public void findExtractAttributesTest(){ List extractAttributes = caseEvidencePropertyService.findExtractAttributes("7"); System.out.println(JSONUtil.toJsonStr(extractAttributes)); } @Test public void saveEvidencePropertyTest() { List attributesDtoList = new ArrayList<>(); NotePromptExtractAttributesDto attributesDto = new NotePromptExtractAttributesDto(); attributesDto.setAttrName("甲方"); attributesDto.setAttrValue("甲方-value"); attributesDto.setAttrValueType("1"); attributesDtoList.add(attributesDto); NotePromptExtractAttributesDto attributesDto2 = new NotePromptExtractAttributesDto(); attributesDto2.setAttrName("资质要求"); attributesDto2.setAttrValue("资质要求-value"); attributesDto2.setAttrValueType("1"); attributesDtoList.add(attributesDto2); NotePromptExtractAttributesDto attributesDto3 = new NotePromptExtractAttributesDto(); attributesDto3.setAttrName("签订日期"); attributesDto3.setAttrValue("2024-10-15"); attributesDto3.setAttrValueType("3"); attributesDtoList.add(attributesDto3); caseEvidencePropertyService.saveEvidenceProperty("1111","7", attributesDtoList); } @Test public void updateEvidencePropertyTest() { List attributesDtoList = new ArrayList<>(); NotePromptExtractAttributesDto attributesDto = new NotePromptExtractAttributesDto(); attributesDto.setAttrName("甲方"); attributesDto.setAttrValue("甲方-value-update"); attributesDto.setAttrValueType("1"); attributesDtoList.add(attributesDto); NotePromptExtractAttributesDto attributesDto2 = new NotePromptExtractAttributesDto(); attributesDto2.setAttrName("资质要求"); attributesDto2.setAttrValue("资质要求-value-update"); attributesDto2.setAttrValueType("1"); attributesDtoList.add(attributesDto2); NotePromptExtractAttributesDto attributesDto3 = new NotePromptExtractAttributesDto(); attributesDto3.setAttrName("签订日期"); attributesDto3.setAttrValue("2024-10-15"); attributesDto3.setAttrValueType("3"); attributesDtoList.add(attributesDto3); caseEvidencePropertyService.updateEvidenceProperty("1111","7", attributesDtoList); } @Test public void listEvidencePropertyTest() { Map> evidenceProperty = caseEvidencePropertyService.listEvidenceProperty("7", List.of("1111")); System.out.println(JSONUtil.toJsonStr(evidenceProperty)); } @Test public void deleteEvidencePropertyTest() { caseEvidencePropertyService.deleteEvidenceProperty("1111", "7"); } @Autowired ModelIndexService modelIndexService; @Autowired ModelCaseService modelCaseService; @Test public void initCaseKnowledgeBaseTest(){ modelCaseService.initCaseKnowledgeBase("1825347949617033217"); } @Test public void migrateRecordKnowledgeBaseTest(){ modelCaseService.migrateRecordKnowledgeBase(); } @Autowired private RecordSplitProcessService recordSplitProcessService; @Test public void replacePronounTest() throws InterruptedException { List noteRecordSplits = new ArrayList<>(); NoteRecordSplit noteRecordSplit = new NoteRecordSplit(); noteRecordSplit.setQuestion("办案民警问: 你给耿彪出售的车辆你们是否签订相关的协议?"); noteRecordSplit.setAnswer("裴金禄答: 我给耿彪出售的车辆一部分签订了车辆报废车辆转让协议,一部没有签订。"); noteRecordSplits.add(noteRecordSplit); NoteRecordSplit noteRecordSplit1 = new NoteRecordSplit(); noteRecordSplit1.setQuestion("办案民警问: 耿彪和你签订协议的时候耿彪是否问你要这些车辆的手续? "); noteRecordSplit1.setAnswer("裴金禄答: 耿彪没有要,"); noteRecordSplits.add(noteRecordSplit1); NoteRecordSplit noteRecordSplit2 = new NoteRecordSplit(); noteRecordSplit2.setQuestion("办案民警问: 既然耿彪都没有和你要你车辆的手续,他是否清楚这些车辆都是不合法渠道来的? "); noteRecordSplit2.setAnswer("裴金禄答: 我不知道耿彪是否清楚,我给他撒谎没有手续,耿彪也就默认再也没有和我要返回结果:"); noteRecordSplits.add(noteRecordSplit2); NoteRecordSplit noteRecordSplit3 = new NoteRecordSplit(); noteRecordSplit3.setQuestion("办案民警问: 你们是在什么地方签订? "); noteRecordSplit3.setAnswer("裴金禄答: 我每次都是在他的车上签订的车辆转让协议,具体签订地方我不知道,我只知道都是在沙坡头区城里面"); noteRecordSplits.add(noteRecordSplit3); List noteRecordSplits1 = recordSplitProcessService.replacePronoun(noteRecordSplits); System.out.println("执行完成"); } }