Merge branch 'release' of http://112.74.196.215:3000/backend/mo-daq-all into develop

mo-daq/db/init.sql

@@ -94,7 +94,7 @@ create table rz_market_operate_ppwfund.mo_report_fund_info
     id           int auto_increment comment '主键，自动递增'
         primary key,
     file_id      int                                not null,
-    fund_name    varchar(255)                       not null comment '基金的名称',
+    fund_name    varchar(255)                       null comment '基金的名称',
     fund_code    varchar(50)                        null comment '基金的唯一识别代码',
     company_name varchar(255)                       null comment '基金管理人的名称',
     currency     varchar(20)                        null comment '基金交易使用的货币种类',
@@ -111,7 +111,7 @@ create table rz_market_operate_ppwfund.mo_report_investor_info
     id                 int auto_increment comment '主键，自动递增'
         primary key,
     file_id            int                                not null,
-    investor_name      varchar(255)                       not null comment '投资人的姓名',
+    investor_name      varchar(255)                       null comment '投资人的姓名',
     investor_type      varchar(50)                        null comment '投资人的类别（例如：个人、机构）',
     certificate_type   varchar(50)                        null comment '证件类型（例如：身份证、护照）',
     certificate_number varchar(50)                        null comment '投资人证件号码',

mo-daq/src/main/java/com/smppw/modaq/application/components/CustomPDFTextStripper.java

@@ -25,7 +25,7 @@ public class CustomPDFTextStripper extends PDFTextStripper {
         for (TextPosition textPosition : textPositions) {
             float[][] values = textPosition.getTextMatrix().getValues();
             double degrees = Math.toDegrees(Math.atan2(values[0][1], values[0][0]));
-            if (Math.abs(degrees % 90 - 0.01) > 0) {
+            if (degrees % 90. > 0.1) {
                 heights.add(textPosition.getHeight());
             }
         }
@@ -45,7 +45,7 @@ public class CustomPDFTextStripper extends PDFTextStripper {
             float[][] values = textPosition.getTextMatrix().getValues();
             double degrees = Math.toDegrees(Math.atan2(values[0][1], values[0][0]));
             float height = textPosition.getHeight();
-            newTexts.add(degrees == 0. && !heights.contains(height) ? textPosition.getUnicode() : WATERMARK_REPLACE);
+            newTexts.add(degrees % 90. <= 0.1 && !heights.contains(height) ? textPosition.getUnicode() : WATERMARK_REPLACE);
         }
         super.writeString(String.join(StrUtil.EMPTY, newTexts));
     }

mo-daq/src/main/java/com/smppw/modaq/application/components/CustomTabulaTextStripper.java

@@ -55,7 +55,7 @@ public class CustomTabulaTextStripper extends TextStripper {
         for (TextPosition textPosition : textPositions) {
             float[][] values = textPosition.getTextMatrix().getValues();
             double degrees = Math.toDegrees(Math.atan2(values[0][1], values[0][0]));
-            if (!Objects.equals(degrees % 90, 0d)) {
+            if (degrees % 90. > 0.1) {
                 rotationTexts.add(textPosition);
             }
         }

mo-daq/src/main/java/com/smppw/modaq/application/components/ReportParseUtils.java

@@ -4,6 +4,7 @@ import cn.hutool.core.collection.ListUtil;
 import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.http.HttpUtil;
+import com.smppw.modaq.common.conts.Constants;
 import com.smppw.modaq.common.enums.ReportParseStatus;
 import com.smppw.modaq.common.enums.ReportType;
 import com.smppw.modaq.common.exception.ReportParseException;
@@ -375,7 +376,8 @@ public final class ReportParseUtils {
 //        String filepath = "C:\\Users\\Administrator\\Desktop\\tmp\\(1)投资者交易确认函【申购】_【SZF635】佳岳国债增强私募证券投资基金_20250217_任军.pdf";
 //        String filepath = "C:\\Users\\Administrator\\Desktop\\tmp\\CP080A_优美利赢胜价值1号私募投资基金A_20250217_邓辉_申购确认_20250217131352.pdf";
 //        String filepath = "C:\\Users\\Administrator\\Desktop\\tmp\\宁水德远国债宝私募证券投资基金_青国平(S21002741743)_申购_20241213_基金交易确认单a2604e57e9a12108.sign.pdf";
-        String filepath = "C:\\Users\\Administrator\\Desktop\\tmp\\基金分红交易确认函_SJH876_2025-02-12_戴羽晨_202502130107720842.pdf";
+//        String filepath = "C:\\Users\\Administrator\\Desktop\\tmp\\基金分红交易确认函_SJH876_2025-02-12_戴羽晨_202502130107720842.pdf";
+        String filepath = "C:\\Users\\Administrator\\Desktop\\tmp\\钧富如风7号私募证券投资基金_陈小明_20250221_073544980_申购确认单.pdf";
 //        String filepath = "C:\\Users\\Administrator\\Desktop\\tmp\\SZN224_君之健睿泰私募证券投资基金_郑为民_20250214_申购确认_20250217102704.pdf";
 
 //        String aiParserUtl = "http://localhost:8088/upload-filepath";
@@ -392,21 +394,18 @@ public final class ReportParseUtils {
 //                "}";
 //        System.out.println(content);
 
-//        List<String> textList;
+        List<String> textList;
         // 解析报告和表格
         try (PDDocument document = Loader.loadPDF(new RandomAccessReadBufferedFile(filepath))) {
-//            PDFTextStripper stripper1 = new PDFTextStripper();
-//            String text1 = stripper1.getText(document);
-
-//            // 识别所有文字（去水印后的）
-//            CustomPDFTextStripper stripper = new CustomPDFTextStripper();
-//            stripper.setSortByPosition(true);
-//            String text = stripper.getText(document).replace(Constants.WATERMARK_REPLACE, StrUtil.EMPTY);
-//            textList = StrUtil.split(text, System.lineSeparator());
-//            textList.removeIf(StrUtil::isBlank);
-//            if (textList.isEmpty()) {
-//                throw new ReportParseException(ReportParseStatus.REPORT_IS_SCAN, "");
-//            }
+            // 识别所有文字（去水印后的）
+            CustomPDFTextStripper stripper = new CustomPDFTextStripper();
+            stripper.setSortByPosition(true);
+            String text = stripper.getText(document).replace(Constants.WATERMARK_REPLACE, StrUtil.EMPTY);
+            textList = StrUtil.split(text, System.lineSeparator());
+            textList.removeIf(StrUtil::isBlank);
+            if (textList.isEmpty()) {
+                throw new ReportParseException(ReportParseStatus.REPORT_IS_SCAN, "");
+            }
             // 解析所有表格（单元格字符去水印）
             List<Table> tables = ListUtil.list(true);
 //            BasicExtractionAlgorithm extractionAlgorithm = new BasicExtractionAlgorithm();

mo-daq/src/main/java/com/smppw/modaq/application/components/report/parser/pdf/AbstractPDReportParser.java

@@ -7,7 +7,9 @@ import cn.hutool.core.util.StrUtil;
 import cn.hutool.http.HttpUtil;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
+import com.smppw.modaq.application.components.CustomPDFTextStripper;
 import com.smppw.modaq.application.components.report.parser.AbstractReportParser;
+import com.smppw.modaq.common.conts.Constants;
 import com.smppw.modaq.common.enums.ReportParseStatus;
 import com.smppw.modaq.common.exception.ReportParseException;
 import com.smppw.modaq.domain.dto.report.ReportBaseInfoDTO;
@@ -39,10 +41,10 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
      * 基金信息表格
      */
     protected Table fundInfoTable;
-//    /**
-//     * 去除了水印的所有文本内容
-//     */
-//    protected List<String> textList;
+    /**
+     * 去除了水印的所有文本内容
+     */
+    protected List<String> textList;
 
     @Value("${email.report.ai-parser-url}")
     private String aiParserUrl;
@@ -61,30 +63,35 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
     public T parse(ReportParserParams params) throws IOException, ReportParseException {
         // 先初始化为null
         this.fundInfoTable = null;
-//        this.textList = null;
+        this.textList = null;
         // 初始化
         this.init();
         String filename = params.getFilename();
         String filepath = params.getFilepath();
         // 解析报告和表格
         try (PDDocument document = Loader.loadPDF(new RandomAccessReadBufferedFile(filepath))) {
-//            // 识别所有文字（去水印后的）
-//            CustomPDFTextStripper stripper = new CustomPDFTextStripper();
-//            stripper.setSortByPosition(true);
-//            String text = stripper.getText(document).replace(Constants.WATERMARK_REPLACE, StrUtil.EMPTY);
-//            this.textList = StrUtil.split(text, System.lineSeparator());
-//            this.textList.removeIf(StrUtil::isBlank);
-//            if (this.textList.isEmpty()) {
-//                throw new ReportParseException(ReportParseStatus.REPORT_IS_SCAN, filename);
-//            }
+            // 识别所有文字（去水印后的）
+            CustomPDFTextStripper stripper = new CustomPDFTextStripper();
+            stripper.setSortByPosition(true);
+            String text = stripper.getText(document).replace(Constants.WATERMARK_REPLACE, StrUtil.EMPTY);
+            this.textList = StrUtil.split(text, System.lineSeparator());
+            this.textList.removeIf(StrUtil::isBlank);
+            if (this.textList.isEmpty()) {
+                throw new ReportParseException(ReportParseStatus.REPORT_IS_SCAN, filename);
+            }
             // 解析所有表格（单元格字符去水印）
             List<Table> tables = ListUtil.list(true);
             SpreadsheetExtractionAlgorithm spreadsheetExtractionAlgorithm = new SpreadsheetExtractionAlgorithm();
             // 自定义表格提取工具，去除单元格中的水印文字
             PageIterator pageIterator = new CustomObjectExtractor(document).extract();
+            // 只解析第一页
+            int i = 0;
             while (pageIterator.hasNext()) {
                 Page page = pageIterator.next();
                 List<Table> tableList = spreadsheetExtractionAlgorithm.extract(page);
+                if (i >= 1) {
+                    break;
+                }
                 Integer rows = tableList.stream().map(Table::getRowCount).filter(rowCount -> rowCount >= 1).reduce(0, Integer::sum);
                 if (rows >= 1) {
                     for (Table table : tableList) {
@@ -109,7 +116,7 @@ public abstract class AbstractPDReportParser<T extends ReportData> extends Abstr
                         this.logger.warn("{} ai解析失败，解析结果{}，错误原因：{}", filename, body, ExceptionUtil.stacktraceToString(e));
                     }
                 }
-
+                i++;
             }
             if (tables.isEmpty() && StrUtil.isBlank(this.aiParserContent)) {
                 throw new ReportParseException(ReportParseStatus.REPORT_IS_SCAN, filename);

mo-daq/src/main/java/com/smppw/modaq/application/components/report/parser/pdf/PDLetterReportParser.java

@@ -57,7 +57,11 @@ public class PDLetterReportParser extends AbstractPDReportParser<LetterReportDat
 
     @Override
     protected ReportFundInfoDTO buildFundInfo(ReportParserParams params) {
-        return this.buildDto(params.getFileId(), ReportFundInfoDTO.class, this.allInfoMap);
+        ReportFundInfoDTO fundInfo = this.buildDto(params.getFileId(), ReportFundInfoDTO.class, this.allInfoMap);
+        if (CollUtil.isNotEmpty(this.textList) && fundInfo.getFundName() == null) {
+            fundInfo.setFundName(this.textList.get(0));
+        }
+        return fundInfo;
     }
 
     @Override

mo-daq/src/main/java/com/smppw/modaq/application/util/EmailUtil.java

@@ -13,17 +13,21 @@ import com.smppw.modaq.domain.dto.MailboxInfoDTO;
 import com.smppw.modaq.infrastructure.util.ExcelUtil;
 import com.smppw.modaq.infrastructure.util.FileUtil;
 import com.sun.mail.imap.IMAPStore;
-import jakarta.activation.DataHandler;
-import jakarta.mail.*;
-import jakarta.mail.internet.*;
-import jakarta.mail.util.ByteArrayDataSource;
+import jakarta.mail.Message;
+import jakarta.mail.MessagingException;
+import jakarta.mail.Session;
+import jakarta.mail.Store;
+import jakarta.mail.internet.MimeBodyPart;
+import jakarta.mail.internet.MimeMultipart;
+import jakarta.mail.internet.MimeUtility;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 
 /**
  * @author mozuwen
@@ -51,7 +55,7 @@ public class EmailUtil {
         String emailTitle = message.getSubject();
         String emailDate = DateUtil.format(message.getSentDate(), DateConst.YYYYMMDDHHMMSS24);
         String emailDateStr = DateUtil.format(message.getSentDate(), DateConst.YYYYMMDD);
-        String filePath = path + "/" + emailAddress + "/" + emailDateStr + "/";
+        String filePath = path + File.separator + emailAddress + File.separator + emailDateStr + File.separator;
 
         MimeMultipart mimeMultipart = (MimeMultipart) message.getContent();
         int length = mimeMultipart.getCount();
@@ -59,50 +63,54 @@ public class EmailUtil {
         for (int i = 0; i < length; i++) {
             EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO();
             MimeBodyPart part = (MimeBodyPart) mimeMultipart.getBodyPart(i);
-//            Object partContent = part.getContent();
+            Object partContent = part.getContent();
             String contentClass = part.getContent().getClass().getSimpleName();
             // 1.邮件正文
-            if ("String".equals(contentClass)) {
-//                // 文件名 = 邮件主题 + 邮件日期
-//                String fileName = emailTitle + "_" + emailDate + ".html";
-//                String content = partContent.toString();
-//                emailContentInfoDTO = collectTextPart(part, content, filePath, fileName);
-            } else if ("BASE64DecoderStream".equals(contentClass)) {
-                if (StrUtil.isNotBlank(part.getFileName())) {
-                    String fileName = MimeUtility.decodeText(part.getFileName());
-                    if (!isSupportedFileType(fileName)) {
-                        continue;
-                    }
-                    emailContentInfoDTO.setFileName(fileName);
+            switch (contentClass) {
+                case "String" -> {
+                    // 文件名 = 邮件主题 + 邮件日期
+                    String fileName = emailTitle + "_" + emailDate + ".html";
+                    String content = partContent.toString();
+                    emailContentInfoDTO = collectTextPart(part, content, filePath, fileName);
+                }
+                case "BASE64DecoderStream" -> {
+                    if (StrUtil.isNotBlank(part.getFileName())) {
+                        String fileName = MimeUtility.decodeText(part.getFileName());
+                        if (isSupportedFileType(fileName)) {
+                            continue;
+                        }
+                        emailContentInfoDTO.setFileName(fileName);
 
-                    String realPath = filePath + emailDate + fileName;
+                        String realPath = filePath + emailDate + fileName;
 
-                    File saveFile = cn.hutool.core.io.FileUtil.file(realPath);
-                    if (!saveFile.exists()) {
-                        if (!saveFile.getParentFile().exists()) {
-                            saveFile.getParentFile().mkdirs();
+                        File saveFile = cn.hutool.core.io.FileUtil.file(realPath);
+                        if (!saveFile.exists()) {
+                            if (!saveFile.getParentFile().exists()) {
+                                saveFile.getParentFile().mkdirs();
+                            }
+                            FileUtil.saveFile(saveFile, part);
+                        } else {
+                            cn.hutool.core.io.FileUtil.del(saveFile);
+                            FileUtil.saveFile(saveFile, part);
+                        }
+                        emailContentInfoDTO.setFilePath(realPath);
+                    }
+                }
+                case "MimeMultipart" -> {
+                    MimeMultipart contentPart = (MimeMultipart) partContent;
+                    int length2 = contentPart.getCount();
+                    for (int i2 = 0; i2 < length2; i2++) {
+                        part = (MimeBodyPart) contentPart.getBodyPart(i2);
+                        partContent = part.getContent();
+                        contentClass = partContent.getClass().getSimpleName();
+                        if ("String".equals(contentClass)) {
+                            // 文件名 = 邮件主题 + 邮件日期
+                            String fileName = emailTitle + "_" + emailDate + ".html";
+                            String content = partContent.toString();
+                            emailContentInfoDTO = collectTextPart(part, content, filePath, fileName);
                         }
-                        FileUtil.saveFile(saveFile, part);
-                    } else {
-                        cn.hutool.core.io.FileUtil.del(saveFile);
-                        FileUtil.saveFile(saveFile, part);
                     }
-                    emailContentInfoDTO.setFilePath(realPath);
                 }
-            } else if ("MimeMultipart".equals(contentClass)) {
-//                MimeMultipart contentPart = (MimeMultipart) partContent;
-//                int length2 = contentPart.getCount();
-//                for (int i2 = 0; i2 < length2; i2++) {
-//                    part = (MimeBodyPart) contentPart.getBodyPart(i2);
-//                    partContent = part.getContent();
-//                    contentClass = partContent.getClass().getSimpleName();
-//                    if ("String".equals(contentClass)) {
-//                        // 文件名 = 邮件主题 + 邮件日期
-//                        String fileName = emailTitle + "_" + emailDate + ".html";
-//                        String content = partContent.toString();
-//                        emailContentInfoDTO = collectTextPart(part, content, filePath, fileName);
-//                    }
-//                }
             }
             String filepath = emailContentInfoDTO.getFilePath();
             if (emailContentInfoDTO.getEmailContent() == null && filepath == null) {
@@ -117,43 +125,43 @@ public class EmailUtil {
         return emailContentInfoDTOList;
     }
 
-    private static List<EmailContentInfoDTO> zipFile(String filepath) {
-        return null;
-    }
+//    private static List<EmailContentInfoDTO> zipFile(String filepath) {
+//        return null;
+//    }
 
     private static boolean isSupportedFileType(String fileName) {
         if (StrUtil.isBlank(fileName)) {
-            return false;
+            return true;
         }
-        return ExcelUtil.isZip(fileName) || ExcelUtil.isExcel(fileName) || ExcelUtil.isPdf(fileName) || ExcelUtil.isHTML(fileName) || ExcelUtil.isRAR(fileName);
+        return !ExcelUtil.isZip(fileName) && !ExcelUtil.isExcel(fileName) && !ExcelUtil.isPdf(fileName) && !ExcelUtil.isHTML(fileName) && !ExcelUtil.isRAR(fileName);
     }
 
-    /**
-     * 根据日期过滤邮件
-     *
-     * @param messages  采集到的邮件
-     * @param startDate 邮件起始日期
-     * @param endDate   邮件截止日期
-     * @return 符合日期的邮件
-     */
-    public static List<Message> filterMessage(Message[] messages, Date startDate, Date endDate) {
-        long startTime = System.currentTimeMillis();
-        List<Message> messageList = CollUtil.newArrayList();
-        if (messages == null) {
-            return messageList;
-        }
-        for (Message message : messages) {
-            try {
-                if (message.getSentDate().compareTo(startDate) >= 0 && message.getSentDate().compareTo(endDate) <= 0) {
-                    messageList.add(message);
-                }
-            } catch (MessagingException e) {
-                throw new RuntimeException(e);
-            }
-        }
-        logger.info("根据日期过滤邮件耗时 -> {}ms", (System.currentTimeMillis() - startTime));
-        return messageList;
-    }
+//    /**
+//     * 根据日期过滤邮件
+//     *
+//     * @param messages  采集到的邮件
+//     * @param startDate 邮件起始日期
+//     * @param endDate   邮件截止日期
+//     * @return 符合日期的邮件
+//     */
+//    public static List<Message> filterMessage(Message[] messages, Date startDate, Date endDate) {
+//        long startTime = System.currentTimeMillis();
+//        List<Message> messageList = CollUtil.newArrayList();
+//        if (messages == null) {
+//            return messageList;
+//        }
+//        for (Message message : messages) {
+//            try {
+//                if (message.getSentDate().compareTo(startDate) >= 0 && message.getSentDate().compareTo(endDate) <= 0) {
+//                    messageList.add(message);
+//                }
+//            } catch (MessagingException e) {
+//                throw new RuntimeException(e);
+//            }
+//        }
+//        logger.info("根据日期过滤邮件耗时 -> {}ms", (System.currentTimeMillis() - startTime));
+//        return messageList;
+//    }
 
     /**
      * 采集邮件正文
@@ -168,7 +176,7 @@ public class EmailUtil {
         EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO();
         try {
             if ((part.getContentType().contains("text/html") || part.getContentType().contains("TEXT/HTML"))) {
-                emailContentInfoDTO.setEmailContent(partContent.toString());
+                emailContentInfoDTO.setEmailContent(partContent);
                 String savePath = filePath + fileName;
                 File saveFile = new File(savePath);
                 if (!saveFile.exists()) {
@@ -181,8 +189,8 @@ public class EmailUtil {
                 String contentType = part.getContentType();
                 String html = partContent.toString();
                 try {
-                    if (contentType.indexOf("charset=") != -1) {
-                        contentType = contentType.substring(contentType.indexOf("charset=") + 8, contentType.length()).replaceAll("\"", "");
+                    if (contentType.contains("charset=")) {
+                        contentType = contentType.substring(contentType.indexOf("charset=") + 8).replaceAll("\"", "");
                         html = html.replace("charset=" + contentType.toLowerCase(), "charset=UTF-8");
                         html = html.replace("charset=" + contentType.toUpperCase(), "charset=UTF-8");
                     }
@@ -201,7 +209,7 @@ public class EmailUtil {
                         return emailContentInfoDTO;
                     }
                     String fileName1 = MimeUtility.decodeText(part.getFileName());
-                    if (!isSupportedFileType(fileName1)) {
+                    if (isSupportedFileType(fileName1)) {
                         return emailContentInfoDTO;
                     }
                     emailContentInfoDTO.setFileName(fileName1);
@@ -302,66 +310,65 @@ public class EmailUtil {
         return props;
     }
 
-    public static void senEmail(MailboxInfoDTO mailboxInfoDTO, String emails, File file, String htmlText, String host, String emailTitle) throws Exception {
-        logger.info("send email begin .........");
-        // 根据Session 构建邮件信息
-        MimeMessage message = new MimeMessage(getSession(mailboxInfoDTO));
-        // 创建邮件发送者地址
-        Address from = new InternetAddress(mailboxInfoDTO.getAccount() + host);
-        String[] emailArr = emails.split(";");
-        Address[] toArr = new Address[emailArr.length];
-        for (int idx = 0; idx < emailArr.length; idx++) {
-            if (StrUtil.isNotBlank(emailArr[idx])) {
-                Address to = new InternetAddress(emailArr[idx]);
-                toArr[idx] = to;
-            }
-        }
-        message.setFrom(from);
-        message.setRecipients(Message.RecipientType.TO, toArr);
-        // 邮件主题
-        message.setSubject(emailTitle);
-        // 邮件容器
-        MimeMultipart mimeMultiPart = new MimeMultipart();
-        // 设置HTML
-        BodyPart bodyPart = new MimeBodyPart();
-        logger.info("组装 htmlText.........");
-        // 邮件内容
-        bodyPart.setContent(htmlText, "text/html;charset=utf-8");
-        //设置附件
-        BodyPart filePart = new MimeBodyPart();
-        filePart.setFileName(file.getName());
-        filePart.setDataHandler(
-                new DataHandler(
-                        new ByteArrayDataSource(
-                                Files.readAllBytes(Paths.get(file.getAbsolutePath())), "application/octet-stream")));
-        mimeMultiPart.addBodyPart(bodyPart);
-        mimeMultiPart.addBodyPart(filePart);
-        message.setContent(mimeMultiPart);
-        message.setSentDate(new Date());
-        // 保存邮件
-        message.saveChanges();
-        // 发送邮件
-        Transport.send(message);
-    }
+//    public static void senEmail(MailboxInfoDTO mailboxInfoDTO, String emails, File file, String htmlText, String host, String emailTitle) throws Exception {
+//        logger.info("send email begin .........");
+//        // 根据Session 构建邮件信息
+//        MimeMessage message = new MimeMessage(getSession(mailboxInfoDTO));
+//        // 创建邮件发送者地址
+//        Address from = new InternetAddress(mailboxInfoDTO.getAccount() + host);
+//        String[] emailArr = emails.split(";");
+//        Address[] toArr = new Address[emailArr.length];
+//        for (int idx = 0; idx < emailArr.length; idx++) {
+//            if (StrUtil.isNotBlank(emailArr[idx])) {
+//                Address to = new InternetAddress(emailArr[idx]);
+//                toArr[idx] = to;
+//            }
+//        }
+//        message.setFrom(from);
+//        message.setRecipients(Message.RecipientType.TO, toArr);
+//        // 邮件主题
+//        message.setSubject(emailTitle);
+//        // 邮件容器
+//        MimeMultipart mimeMultiPart = new MimeMultipart();
+//        // 设置HTML
+//        BodyPart bodyPart = new MimeBodyPart();
+//        logger.info("组装 htmlText.........");
+//        // 邮件内容
+//        bodyPart.setContent(htmlText, "text/html;charset=utf-8");
+//        //设置附件
+//        BodyPart filePart = new MimeBodyPart();
+//        filePart.setFileName(file.getName());
+//        filePart.setDataHandler(
+//                new DataHandler(
+//                        new ByteArrayDataSource(
+//                                Files.readAllBytes(Paths.get(file.getAbsolutePath())), "application/octet-stream")));
+//        mimeMultiPart.addBodyPart(bodyPart);
+//        mimeMultiPart.addBodyPart(filePart);
+//        message.setContent(mimeMultiPart);
+//        message.setSentDate(new Date());
+//        // 保存邮件
+//        message.saveChanges();
+//        // 发送邮件
+//        Transport.send(message);
+//    }
 
-    public static Session getSession(MailboxInfoDTO mailboxInfoDTO) {
-        try {
-            Properties properties = new Properties();
-            properties.put("mail.smtp.host", mailboxInfoDTO.getHost());
-            properties.put("mail.smtp.auth", true);
-            properties.put("mail.smtp.port", mailboxInfoDTO.getPort());
-            properties.put("mail.smtp.ssl.enable", true);
-            final String SSL_FACTORY = "javax.net.ssl.SSLSocketFactory";
-            properties.setProperty("mail.smtp.socketFactory.class", SSL_FACTORY);
-            properties.setProperty("mail.smtp.socketFactory.fallback", "false");
-            properties.setProperty("mail.smtp.socketFactory.port", mailboxInfoDTO.getPort());
-            // 根据邮件的会话属性构造一个发送邮件的Session，
-            JakartaUserPassAuthenticator authenticator = new JakartaUserPassAuthenticator(mailboxInfoDTO.getAccount(), mailboxInfoDTO.getPassword());
-            Session session = Session.getInstance(properties, authenticator);
-            return session;
-        } catch (Exception e) {
-            logger.error("getSession : " + e.getMessage());
-        }
-        return null;
-    }
+//    public static Session getSession(MailboxInfoDTO mailboxInfoDTO) {
+//        try {
+//            Properties properties = new Properties();
+//            properties.put("mail.smtp.host", mailboxInfoDTO.getHost());
+//            properties.put("mail.smtp.auth", true);
+//            properties.put("mail.smtp.port", mailboxInfoDTO.getPort());
+//            properties.put("mail.smtp.ssl.enable", true);
+//            final String SSL_FACTORY = "javax.net.ssl.SSLSocketFactory";
+//            properties.setProperty("mail.smtp.socketFactory.class", SSL_FACTORY);
+//            properties.setProperty("mail.smtp.socketFactory.fallback", "false");
+//            properties.setProperty("mail.smtp.socketFactory.port", mailboxInfoDTO.getPort());
+//            // 根据邮件的会话属性构造一个发送邮件的Session，
+//            JakartaUserPassAuthenticator authenticator = new JakartaUserPassAuthenticator(mailboxInfoDTO.getAccount(), mailboxInfoDTO.getPassword());
+//            return Session.getInstance(properties, authenticator);
+//        } catch (Exception e) {
+//            logger.error("getSession : {}", e.getMessage());
+//        }
+//        return null;
+//    }
 }

mo-daq/src/main/java/com/smppw/modaq/common/enums/ReportParseStatus.java

@@ -1,18 +1,18 @@
 package com.smppw.modaq.common.enums;
 
 public enum ReportParseStatus implements StatusCode {
-    PARSE_FAIL(21000, "定期报告解析错误：{}"),
-    NOT_A_REPORT(21001, "[{}]不是定期报告"),
+    PARSE_FAIL(21000, "定期报告或交易确认单解析错误：{}"),
+    NOT_A_REPORT(21001, "[{}]不是定期报告或交易确认单"),
     REPORT_IS_SCAN(21002, "报告[{}]为扫描件"),
     NO_SUPPORT_TEMPLATE(21003, "报告[{}]是不支持的文件格式"),
     NOT_A_FIXED_FORMAT(21004, "报告[{}]不是基协统一格式"),
 
-    PARSE_FUND_INFO_FAIL(21010, "报告[{}]没有解析到基金基本信息"),
-    PARSE_NAV_INFO_FAIL(21011, "报告[{}]没有解析到基金净值信息"),
-    PARSE_FINANCIAL_INFO_FAIL(21012, "报告[{}]没有解析到基金财务指标信息"),
-    PARSE_INDUSTRY_INFO_FAIL(21013, "报告[{}]没有解析到基金行业配置信息"),
-    PARSE_ASSET_INFO_FAIL(21014, "报告[{}]没有解析到基金资产配置信息"),
-    PARSE_SHARE_INFO_FAIL(21015, "报告[{}]没有解析到基金份额变动信息"),
+//    PARSE_FUND_INFO_FAIL(21010, "报告[{}]没有解析到基金基本信息"),
+//    PARSE_NAV_INFO_FAIL(21011, "报告[{}]没有解析到基金净值信息"),
+//    PARSE_FINANCIAL_INFO_FAIL(21012, "报告[{}]没有解析到基金财务指标信息"),
+//    PARSE_INDUSTRY_INFO_FAIL(21013, "报告[{}]没有解析到基金行业配置信息"),
+//    PARSE_ASSET_INFO_FAIL(21014, "报告[{}]没有解析到基金资产配置信息"),
+//    PARSE_SHARE_INFO_FAIL(21015, "报告[{}]没有解析到基金份额变动信息"),
 
     PARSE_RULE_NO_FUND(21020, "未设置报告解析规则"),
     ;

mo-daq/src/main/java/com/smppw/modaq/domain/service/EmailParseService.java

@@ -30,6 +30,7 @@ import com.smppw.modaq.domain.entity.EmailParseInfoDO;
 import com.smppw.modaq.domain.mapper.EmailFileInfoMapper;
 import com.smppw.modaq.domain.mapper.EmailParseInfoMapper;
 import com.smppw.modaq.infrastructure.util.ExcelUtil;
+import com.smppw.modaq.infrastructure.util.FileUtil;
 import jakarta.mail.*;
 import jakarta.mail.internet.MimeMessage;
 import jakarta.mail.internet.MimeMultipart;
@@ -44,7 +45,6 @@ import org.springframework.stereotype.Service;
 import org.springframework.util.StopWatch;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.util.*;
@@ -123,6 +123,11 @@ public class EmailParseService {
             Iterator<EmailContentInfoDTO> iterator = emailContentInfoDTOList.iterator();
             while (iterator.hasNext()) {
                 EmailContentInfoDTO emailContentInfoDTO = iterator.next();
+                // 正文不用解压附件
+                if (emailContentInfoDTO.getFileName() != null && emailContentInfoDTO.getFileName().endsWith(".html")) {
+                    emailZipFileMap.put(emailContentInfoDTO, ListUtil.empty());
+                    continue;
+                }
                 try {
                     List<EmailZipFileDTO> fundNavDTOList = parseZipEmail(emailContentInfoDTO);
                     emailZipFileMap.put(emailContentInfoDTO, fundNavDTOList);
@@ -179,6 +184,11 @@ public class EmailParseService {
             return;
         }
         for (String dir : extractedDirs) {
+            // 如果邮件类型不满足解析条件则重新根据文件名判断
+            if (!Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)
+                    && !Objects.equals(EmailTypeConst.REPORT_LETTER_EMAIL_TYPE, emailType)) {
+                emailType = EmailUtil.getEmailTypeBySubject(dir, this.getEmailType());
+            }
             File file = new File(dir);
             if (file.isDirectory()) {
                 String[] subDirs = file.list();
@@ -201,14 +211,12 @@ public class EmailParseService {
         return Paths.get(filepath).getParent().resolve(baseName).toString();
     }
 
-
     public void saveRelatedTable(String emailKey, String emailAddress,
                                  Map<EmailContentInfoDTO, List<EmailZipFileDTO>> emailZipFileMap) {
         // python 报告解析接口结果
         List<ParseResult<ReportData>> dataList = ListUtil.list(false);
         for (Map.Entry<EmailContentInfoDTO, List<EmailZipFileDTO>> entry : emailZipFileMap.entrySet()) {
             EmailContentInfoDTO emailContentInfoDTO = entry.getKey();
-            Integer emailType = emailContentInfoDTO.getEmailType();
             Integer emailId = emailContentInfoDTO.getEmailId();
             EmailParseInfoDO emailParseInfoDO = buildEmailParseInfo(emailId, emailAddress, emailContentInfoDTO);
             emailParseInfoDO.setEmailKey(emailKey);
@@ -216,6 +224,9 @@ public class EmailParseService {
             if (emailId == null) {
                 continue;
             }
+            if (emailContentInfoDTO.getFileName() != null && emailContentInfoDTO.getFileName().endsWith(".html")) {
+                continue;
+            }
 
             List<EmailZipFileDTO> zipFiles = entry.getValue();
             if (CollUtil.isNotEmpty(zipFiles)) {
@@ -223,7 +234,7 @@ public class EmailParseService {
                     EmailFileInfoDO emailFile = saveEmailFileInfo(emailId, null, zipFile.getFilename(), zipFile.getFilepath(), null);
                     // 解析结果（可以从python获取或者自行解析）并保存报告
                     ParseResult<ReportData> parseResult = this.parseReportAndHandleResult(emailFile.getId(), zipFile.getFilename(),
-                            zipFile.getFilepath(), emailType, emailFile.getAiFileId());
+                            zipFile.getFilepath(), zipFile.getEmailType(), emailFile.getAiFileId());
                     dataList.add(parseResult);
                 }
             } else {
@@ -232,7 +243,7 @@ public class EmailParseService {
                         emailContentInfoDTO.getFilePath(), emailContentInfoDTO.getAiFileId());
                 // 解析结果（可以从python获取或者自行解析）并保存报告
                 ParseResult<ReportData> parseResult = this.parseReportAndHandleResult(emailFile.getId(), fileName,
-                        emailContentInfoDTO.getFilePath(), emailType, emailFile.getAiFileId());
+                        emailContentInfoDTO.getFilePath(), emailContentInfoDTO.getEmailType(), emailFile.getAiFileId());
                 dataList.add(parseResult);
             }
 
@@ -263,9 +274,10 @@ public class EmailParseService {
         ParseResult<ReportData> result = new ParseResult<>();
         if ((!Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)
                 && !Objects.equals(EmailTypeConst.REPORT_LETTER_EMAIL_TYPE, emailType))
-                || StrUtil.isBlank(fileName)) {
+                || StrUtil.isBlank(fileName)
+                || fileName.endsWith(".html")) {
             result.setStatus(ReportParseStatus.NOT_A_REPORT.getCode());
-            result.setMsg(ReportParseStatus.NOT_A_REPORT.getMsg());
+            result.setMsg(StrUtil.format(ReportParseStatus.NOT_A_REPORT.getMsg(), fileName));
             return result;
         }
         Pattern pattern = Pattern.compile("[A-Z0-9]{6}");
@@ -305,9 +317,9 @@ public class EmailParseService {
             result.setMsg("报告解析成功");
             result.setData(reportData);
         } catch (ReportParseException e) {
-            log.error("解析失败\n{}", StrUtil.format(e.getMsg(), filepath));
+            log.error("解析失败\n{}", StrUtil.format(e.getMsg(), fileName));
             result.setStatus(e.getCode());
-            result.setMsg(StrUtil.format(e.getMsg(), filepath));
+            result.setMsg(StrUtil.format(e.getMsg(), fileName));
         } catch (Exception e) {
             log.error("解析错误\n{}", ExceptionUtil.stacktraceToString(e));
             result.setStatus(ReportParseStatus.PARSE_FAIL.getCode());
@@ -371,11 +383,11 @@ public class EmailParseService {
             emailParseInfoMapper.updateParseTime(emailParseInfoDO.getId(), emailParseInfoDO.getParseDate());
             return emailParseInfoDO.getId();
         }
-        // 根据邮件发送人、邮件地址、邮箱日期、主题找到是否已经存在的记录（不管是否成功），已存在就不解析了
-        EmailParseInfoDO temp = this.emailParseInfoMapper.searchEmail(emailParseInfoDO);
-        if (temp != null) {
-            return null;
-        }
+//        // 根据邮件发送人、邮件地址、邮箱日期、主题找到是否已经存在的记录（不管是否成功），已存在就不解析了
+//        EmailParseInfoDO temp = this.emailParseInfoMapper.searchEmail(emailParseInfoDO);
+//        if (temp != null) {
+//            return null;
+//        }
         emailParseInfoMapper.insert(emailParseInfoDO);
         return emailParseInfoDO.getId();
     }
@@ -406,8 +418,9 @@ public class EmailParseService {
 //        String report = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getReport()) ? emailTypeRuleDO.getReport() : emailRuleConfig.getReport();
 //        emailTypeMap.put(EmailTypeConst.VALUATION_EMAIL_TYPE, Arrays.stream(valuation.split(",")).toList());
 //        emailTypeMap.put(EmailTypeConst.NAV_EMAIL_TYPE, Arrays.stream(nav.split(",")).toList());
-        emailTypeMap.put(EmailTypeConst.REPORT_EMAIL_TYPE, ListUtil.toList("月报", "周报", "月度报告"));
-        emailTypeMap.put(EmailTypeConst.REPORT_LETTER_EMAIL_TYPE, ListUtil.toList("确认单", "确认函", "确认"));
+//        emailTypeMap.put(EmailTypeConst.REPORT_EMAIL_TYPE, ListUtil.toList("月报", "周报", "月度报告"));
+        emailTypeMap.put(EmailTypeConst.REPORT_LETTER_EMAIL_TYPE,
+                ListUtil.toList("确认单", "确认函", "交易确认数据", "赎回确认", "申购确认", "分红确认", "确认表"));
         return emailTypeMap;
     }
 
@@ -461,32 +474,32 @@ public class EmailParseService {
                 if (content instanceof MimeMultipart) {
                     emailContentInfoDTOList = EmailUtil.collectMimeMultipart(message, mailboxInfoDTO.getAccount(), path);
                 }
-//                // 2.邮件只有正文
-//                if (content instanceof String) {
-//                    EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO();
-//                    emailContentInfoDTO.setEmailContent(content.toString());
-//                    emailContentInfoDTO.setEmailDate(emailDateStr);
-//                    emailContentInfoDTO.setEmailTitle(message.getSubject());
-//                    String fileName = message.getSubject() + DateUtil.format(emailDate, DateConst.YYYYMMDDHHMMSS24);
-//                    String filePath = path + mailboxInfoDTO.getAccount() + "/" + DateUtil.format(emailDate, DateConst.YYYY_MM_DD) + "/" + fileName + ".html";
-//                    File saveFile = new File(filePath);
-//                    saveFile.setReadable(true);
-//                    if (!saveFile.exists()) {
-//                        if (!saveFile.getParentFile().exists()) {
-//                            saveFile.getParentFile().mkdirs();
-//                            saveFile.getParentFile().setExecutable(true);
-//                        }
-//                    }
-//                    FileUtil.writeFile(filePath, content.toString());
-//                    emailContentInfoDTO.setFilePath(filePath);
-//                    emailContentInfoDTO.setFileName(fileName);
-//                    emailContentInfoDTOList.add(emailContentInfoDTO);
-//                }
-                if (CollUtil.isNotEmpty(emailContentInfoDTOList)) {
-                    // 估值表或定期报告邮件不展示正文html文件
-                    if (emailType.equals(EmailTypeConst.VALUATION_EMAIL_TYPE) || emailType.equals(EmailTypeConst.REPORT_EMAIL_TYPE)) {
-                        emailContentInfoDTOList = emailContentInfoDTOList.stream().filter(e -> !ExcelUtil.isHTML(e.getFilePath())).toList();
+                // 2.邮件只有正文
+                if (content instanceof String) {
+                    EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO();
+                    emailContentInfoDTO.setEmailContent(content.toString());
+                    emailContentInfoDTO.setEmailDate(emailDateStr);
+                    emailContentInfoDTO.setEmailTitle(message.getSubject());
+                    String fileName = message.getSubject() + DateUtil.format(emailDate, DateConst.YYYYMMDDHHMMSS24);
+                    String filePath = path + mailboxInfoDTO.getAccount() + File.separator + DateUtil.format(emailDate, DateConst.YYYY_MM_DD) + File.separator + fileName + ".html";
+                    File saveFile = new File(filePath);
+                    saveFile.setReadable(true);
+                    if (!saveFile.exists()) {
+                        if (!saveFile.getParentFile().exists()) {
+                            saveFile.getParentFile().mkdirs();
+                            saveFile.getParentFile().setExecutable(true);
+                        }
                     }
+                    FileUtil.writeFile(filePath, content.toString());
+                    emailContentInfoDTO.setFilePath(filePath);
+                    emailContentInfoDTO.setFileName(fileName);
+                    emailContentInfoDTOList.add(emailContentInfoDTO);
+                }
+                if (CollUtil.isNotEmpty(emailContentInfoDTOList)) {
+//                    // 估值表或定期报告邮件不展示正文html文件
+//                    if (emailType.equals(EmailTypeConst.VALUATION_EMAIL_TYPE) || emailType.equals(EmailTypeConst.REPORT_EMAIL_TYPE)) {
+//                        emailContentInfoDTOList = emailContentInfoDTOList.stream().filter(e -> !ExcelUtil.isHTML(e.getFilePath())).toList();
+//                    }
                     emailContentInfoDTOList.forEach(e -> {
                         e.setEmailType(emailType);
                         e.setSenderEmail(senderEmail);

mo-daq/src/main/java/com/smppw/modaq/infrastructure/util/ExcelUtil.java

@@ -44,11 +44,23 @@ public class ExcelUtil {
             destFile.mkdirs();
         }
 
+
         try (BufferedInputStream fis = new BufferedInputStream(new FileInputStream(zipFilePath));
              ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream(fis)) {
             ArchiveEntry entry;
+            int i = 1;
             while ((entry = ais.getNextEntry()) != null) {
-                File entryFile = new File(destFilePath, entry.getName());
+                String name = entry.getName();
+                File entryFile;
+                try {
+                    entryFile = FileUtil.file(destFilePath, name);
+                } catch (Exception e) {
+                    String zipFilename = FileUtil.getName(destFilePath);
+                    String ext = FileUtil.extName(name);
+                    name = zipFilename + "_" + i + "." + ext;
+                    entryFile = FileUtil.file(destFilePath, name);
+                    i++;
+                }
                 if (entry.isDirectory()) {
                     entryFile.mkdirs();
                 } else {
@@ -67,12 +79,18 @@ public class ExcelUtil {
         List<String> fileList = new ArrayList<>();
         // 创建Archive对象，用于读取rar压缩文件格式
         try {
+            String zipFilename = FileUtil.getName(inputFilePath);
             Archive archive = new Archive(new FileInputStream(inputFilePath));
             // 读取压缩文件中的所有子目录或子文件（FileHeader对象）
             List<FileHeader> fileHeaderList = archive.getFileHeaders();
+            int i = 1;
             // 遍历子目录和子文件
             for (FileHeader fd : fileHeaderList) {
-                File f = FileUtil.file(outputDirPath + File.separator + fd.getFileName());
+                String fileName = fd.getFileName();
+                String ext = FileUtil.extName(fileName);
+                fileName = zipFilename + "_" + i + "." + ext;
+                i++;
+                File f = FileUtil.file(outputDirPath + File.separator + fileName);
                 if (fd.isDirectory()) {
                     // 创建新子目录
                     f.mkdirs();

mo-daq/src/test/java/com/smppw/modaq/MoDaqApplicationTests.java

@@ -35,8 +35,8 @@ public class MoDaqApplicationTests {
     @Test
     public void reportTest() {
         MailboxInfoDTO emailInfoDTO = this.buildMailbox("xx@simuwang.com", "**");
-        Date startDate = DateUtil.parse("2025-02-17 15:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2025-02-17 16:08:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2025-02-25 14:56:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2025-02-25 14:58:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {