mo-daq/src/main/java/com/smppw/modaq/application/components/ReportParseUtils.java

@@ -448,9 +448,11 @@ public final class ReportParseUtils {
             if (Objects.equals(EmailTypeConst.REPORT_LETTER_EMAIL_TYPE, emailType)
                     || StrUtil.containsAny(text, ReportType.LETTER.getPatterns())) {
                 reportType = ReportType.LETTER;
-            } else if (StrUtil.containsAny(text, ReportType.WEEKLY.getPatterns())) {
+            } else if (Objects.equals(EmailTypeConst.REPORT_WEEKLY_TYPE, emailType)
+                    || StrUtil.containsAny(text, ReportType.WEEKLY.getPatterns())) {
                 reportType = ReportType.WEEKLY;
-            } else if (StrUtil.containsAny(text, ReportType.OTHER.getPatterns())) {
+            } else if (Objects.equals(EmailTypeConst.REPORT_OTHER_TYPE, emailType)
+                    || StrUtil.containsAny(text, ReportType.OTHER.getPatterns())) {
                 reportType = ReportType.OTHER;
             }
             return reportType;
@@ -466,6 +468,10 @@ public final class ReportParseUtils {
             // 特殊的月报（当季度->年度->月度报告无法识别时）
             reportType = ReportType.MONTHLY;
         }
+        // 如果type=3并且没有识别出报告类型则默认为月报来解析
+        if (reportType == null && Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) {
+            reportType = ReportType.MONTHLY;
+        }
         return reportType;
     }
 

mo-daq/src/main/java/com/smppw/modaq/application/components/report/parser/ai/AbstractAIReportParser.java

@@ -31,7 +31,9 @@ public abstract class AbstractAIReportParser<T extends ReportData> extends Abstr
     private String aiParserUrl;
 
 //    protected String aiFileId;
-
+    /**
+     * 报告AI解析结果，如果有嵌套的数据结构要平铺出来
+     */
     protected Map<String, Object> allInfoMap;
 
     public AbstractAIReportParser(EmailFieldMappingMapper fieldMappingMapper) {
@@ -151,7 +153,7 @@ public abstract class AbstractAIReportParser<T extends ReportData> extends Abstr
                 this.handleAiResult(result);
             }
         } catch (ReportParseException e) {
-            this.logger.warn("{} ai解析失败，解析结果{}，错误原因：{}", filename, body, ExceptionUtil.stacktraceToString(e));
+            this.logger.warn("报告{} ai解析失败，解析结果{}，错误原因：{}", filename, body, ExceptionUtil.stacktraceToString(e));
             throw e;
         } catch (IORuntimeException e) {
             throw new ReportParseException(ReportParseStatus.AI_NOT_FOUND);

mo-daq/src/main/java/com/smppw/modaq/application/components/report/writer/AbstractReportWriter.java

@@ -1,5 +1,6 @@
 package com.smppw.modaq.application.components.report.writer;
 
+import cn.hutool.core.util.StrUtil;
 import com.smppw.modaq.domain.dto.report.ReportBaseInfoDTO;
 import com.smppw.modaq.domain.dto.report.ReportData;
 import com.smppw.modaq.domain.dto.report.ReportFundInfoDTO;
@@ -43,7 +44,7 @@ public abstract class AbstractReportWriter<T extends ReportData> implements Repo
 
     private void saveFundInfo(T reportData) {
         ReportFundInfoDTO fundInfo = reportData.getFundInfo();
-        if (fundInfo != null) {
+        if (fundInfo != null && StrUtil.isAllNotBlank(fundInfo.getFundName(), fundInfo.getFundCode())) {
             this.fundInfoMapper.insert(fundInfo.toEntity());
         }
     }

mo-daq/src/main/java/com/smppw/modaq/application/service/EmailParseApiServiceImpl.java

@@ -101,11 +101,13 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
     public List<UploadReportResult> uploadUrlReport(UploadUrlReportParams params) {
         List<UploadReportParams.ReportInfo> reportInfos = ListUtil.list(false);
         List<UploadUrlReportParams.UrlReportInfo> urlReportInfos = params.getReportInfos();
+        List<String> contents = ListUtil.list(false);
         for (UploadUrlReportParams.UrlReportInfo urlReportInfo : urlReportInfos) {
             String reportUrl = urlReportInfo.getReportUrl();
+            contents.add(reportUrl);
             String filename;
             if (StrUtil.isNotBlank(getFileExtension(urlReportInfo.getReportName()))) {
-                filename = reportUrl;
+                filename = urlReportInfo.getReportName();
             } else {
                 filename = urlReportInfo.getReportName() + "." + getFileExtension(reportUrl);
             }
@@ -124,6 +126,7 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
         }
         UploadReportParams req = new UploadReportParams();
         req.setReportInfos(reportInfos);
+        req.setContent(String.join(",\n", contents));
         if (StrUtil.isBlank(params.getTitle())) {
             req.setTitle("报告上传解析-url");
         } else {

mo-daq/src/main/java/com/smppw/modaq/common/conts/DateConst.java

@@ -1,12 +0,0 @@
-package com.smppw.modaq.common.conts;
-
-public class DateConst {
-
-    public final static String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss";
-    public final static String YYYY_MM_DD_HH_MM = "yyyy-MM-dd HH:mm";
-    public final static String YYYY_MM_DD = "yyyy-MM-dd";
-    public final static String YYYYMMDD = "yyyyMMdd";
-    public final static String YYYYMMDDHHMMSS = "yyyyMMddHHmmssSSS";
-    public final static String YYYYMMDDHHMMSS24 = "yyyyMMddHHmmss";
-
-}

mo-daq/src/main/java/com/smppw/modaq/domain/dto/EmailZipFileDTO.java

@@ -13,18 +13,21 @@ public class EmailZipFileDTO {
     private final String filename;
     private final String filepath;
     private final long fileSize;
+    private final String extName;
+
     // pdf文件的密码（需要密码时）
-    @Setter
     private String pdfPwd;
     @Setter
     private Integer emailType;
-
-    private final String extName;
-
     @Setter
     private Integer fileId;
+    @Setter
+    private Integer parseStatus;
+    @Setter
+    private String failReason;
 
     public EmailZipFileDTO(String emailTitle, String filepath, Integer emailType) {
+        this.parseStatus = 1;
         File file = FileUtil.file(filepath);
         this.emailTitle = emailTitle;
         this.filepath = filepath;
@@ -35,6 +38,7 @@ public class EmailZipFileDTO {
     }
 
     public EmailZipFileDTO(String emailTitle, EmailContentInfoDTO emailDto) {
+        this.parseStatus = 1;
         this.emailTitle = emailTitle;
         this.filepath = emailDto.getFilePath();
         this.emailType = emailDto.getEmailType();
@@ -45,18 +49,20 @@ public class EmailZipFileDTO {
     }
 
     public EmailZipFileDTO(String emailTitle, UploadReportParams.ReportInfo uploadReportInfo) {
+        this.parseStatus = 1;
+        File file = FileUtil.file(uploadReportInfo.getReportPath());
         this.emailTitle = emailTitle;
         this.filepath = uploadReportInfo.getReportPath();
         this.emailType = uploadReportInfo.getReportType();
         this.filename = uploadReportInfo.getReportName();
         this.pdfPwd = uploadReportInfo.getPdfPwd();
-        this.fileSize = FileUtil.size(FileUtil.file(this.filepath));
-        this.extName = FileUtil.extName(filepath);
+        this.fileSize = FileUtil.size(file);
+        this.extName = FileUtil.extName(file);
     }
 
     @Override
     public String toString() {
-        return "EmailZipFileDTO{" +
+        return "{" +
                 "emailTitle='" + emailTitle + '\'' +
                 ", filename='" + filename + '\'' +
                 ", filepath='" + filepath + '\'' +

mo-daq/src/main/java/com/smppw/modaq/domain/dto/UploadReportParams.java

@@ -14,6 +14,7 @@ import java.util.List;
 @Getter
 public class UploadReportParams {
     private String title;
+    private String content;
     private List<ReportInfo> reportInfos;
 
     public UploadReportParams() {

mo-daq/src/main/java/com/smppw/modaq/domain/entity/EmailParseInfoDO.java

@@ -43,6 +43,10 @@ public class EmailParseInfoDO {
     @TableField(value = "email_title")
     private String emailTitle;
     /**
+     * 邮件内容（html格式）
+     */
+    private String emailContent;
+    /**
      * 邮件类型，1-净值，2-估值表，3-定期报告
      */
     @TableField(value = "email_type")

mo-daq/src/main/java/com/smppw/modaq/domain/mapper/EmailFileInfoMapper.java

@@ -17,7 +17,7 @@ public interface EmailFileInfoMapper extends BaseMapper<EmailFileInfoDO> {
 //
 //    List<EmailFileInfoDO> getEmailFileByEmailId(@Param("emailId") Integer emailId);
 
-    List<EmailFileInfoDO> queryByEmailId(@Param("emailId") Integer emailId);
+//    List<EmailFileInfoDO> queryByEmailId(@Param("emailId") Integer emailId);
 
 //    List<FundFileInfoVO> searchFundFileInfo(FundFilePageQuery fundFilePageQuery);
 //

mo-daq/src/main/java/com/smppw/modaq/domain/service/EmailParseService.java

@@ -2,6 +2,7 @@ package com.smppw.modaq.domain.service;
 
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.date.DatePattern;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.io.FileUtil;
@@ -18,7 +19,6 @@ import com.smppw.modaq.application.components.report.writer.ReportWriter;
 import com.smppw.modaq.application.components.report.writer.ReportWriterFactory;
 import com.smppw.modaq.application.util.EmailUtil;
 import com.smppw.modaq.common.conts.Constants;
-import com.smppw.modaq.common.conts.DateConst;
 import com.smppw.modaq.common.conts.EmailParseStatusConst;
 import com.smppw.modaq.common.conts.EmailTypeConst;
 import com.smppw.modaq.common.enums.ReportMonthlyType;
@@ -144,8 +144,8 @@ public class EmailParseService {
             emailTypes = ListUtil.of(EmailTypeConst.REPORT_LETTER_EMAIL_TYPE);
         }
         if (log.isInfoEnabled()) {
-            log.info("开始邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate,
-                    DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
+            log.info("开始邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO,
+                    DateUtil.formatDateTime(startDate), DateUtil.formatDateTime(endDate));
         }
         Map<String, List<EmailContentInfoDTO>> emailContentMap;
         try {
@@ -156,21 +156,21 @@ public class EmailParseService {
         }
         if (MapUtil.isEmpty(emailContentMap)) {
             log.warn("未采集到邮件 -> 邮箱配置信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO,
-                    DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
+                    DateUtil.formatDateTime(startDate), DateUtil.formatDateTime(endDate));
             return;
         }
         for (Map.Entry<String, List<EmailContentInfoDTO>> emailEntry : emailContentMap.entrySet()) {
-            List<EmailContentInfoDTO> emailContentInfoDTOList = emailEntry.getValue();
-            if (CollUtil.isEmpty(emailContentInfoDTOList)) {
+            List<EmailContentInfoDTO> dtoList = emailEntry.getValue();
+            if (CollUtil.isEmpty(dtoList)) {
                 log.warn("未采集到正文或附件");
                 continue;
             }
-            EmailContentInfoDTO dto = emailContentInfoDTOList.get(0);
+            EmailContentInfoDTO dto = dtoList.get(0);
             String emailTitle = dto.getEmailTitle();
             if (log.isInfoEnabled()) {
                 log.info("开始解析邮件数据 -> 邮件主题:{},邮件日期:{}", emailTitle, dto.getEmailDate());
             }
-            Long totalSize = emailContentInfoDTOList.stream().map(EmailContentInfoDTO::getFileSize).reduce(0L, Long::sum);
+            Long totalSize = dtoList.stream().map(EmailContentInfoDTO::getFileSize).reduce(0L, Long::sum);
             String errMsg = null;
             int status = 1;
             List<EmailZipFileDTO> emailFileList = ListUtil.list(false);
@@ -179,7 +179,7 @@ public class EmailParseService {
                 status = 0;
                 errMsg = "邮件中存在超大附件，需要手动处理该邮件";
             } else {
-                for (EmailContentInfoDTO emailDto : emailContentInfoDTOList) {
+                for (EmailContentInfoDTO emailDto : dtoList) {
                     // 正文不用解压附件
                     if (emailDto.getFileName() != null && emailDto.getFileName().endsWith(Constants.FILE_HTML)) {
                         continue;
@@ -188,12 +188,12 @@ public class EmailParseService {
                         emailFileList.addAll(this.parseZipEmail(emailDto));
                     } catch (IOException e) {
                         log.error("邮件{} 压缩包解压失败:{}", emailTitle, ExceptionUtil.stacktraceToString(e));
-                        status = 0;
-                        errMsg = "压缩包解压失败";
+                        EmailZipFileDTO zipFileDTO = new EmailZipFileDTO(emailTitle, emailDto);
+                        zipFileDTO.setParseStatus(0);
+                        zipFileDTO.setFailReason("压缩包解压失败");
+                        emailFileList.add(zipFileDTO);
                     } catch (Exception e) {
                         log.error("邮件{} 堆栈信息:{}", emailTitle, ExceptionUtil.stacktraceToString(e));
-                        status = 0;
-                        errMsg = "内部错误";
                     }
                 }
                 // 重新判断类型
@@ -222,14 +222,18 @@ public class EmailParseService {
             Integer emailId = this.saveEmailParseInfo(emailDo);
             // 保存附件（解压后的）
             for (EmailZipFileDTO zipFile : emailFileList) {
-                EmailFileInfoDO emailFile = this.saveEmailFileInfo(emailId, zipFile.getFilename(), zipFile.getFilepath());
+                EmailFileInfoDO emailFile = this.saveEmailFileInfo(emailId, zipFile);
+                if (!Objects.equals(1, zipFile.getParseStatus()) || StrUtil.isNotBlank(zipFile.getFailReason())) {
+                    emailFile.setParseStatus(zipFile.getParseStatus());
+                    emailFile.setFailReason(zipFile.getFailReason());
+                }
                 zipFile.setFileId(emailFile.getId());
             }
             if (CollUtil.isNotEmpty(emailFileList)) {
                 // 保存相关信息 -> 邮件信息表,邮件文件表,邮件净值表,邮件规模表,基金净值表
                 this.saveRelatedTable(emailId, emailInfo);
                 log.info("结束邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", emailEntry.getValue(),
-                        DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
+                        DateUtil.formatDateTime(startDate), DateUtil.formatDateTime(endDate));
             }
         }
     }
@@ -355,35 +359,41 @@ public class EmailParseService {
      * @return /
      */
     public List<UploadReportResult> uploadReportResults(UploadReportParams params) {
-        List<ParseResult<ReportData>> dataList = ListUtil.list(false);
         String emailTitle = params.getTitle();
         List<UploadReportParams.ReportInfo> reportInfos = params.getReportInfos();
         List<EmailZipFileDTO> dtos = ListUtil.list(false);
         for (UploadReportParams.ReportInfo e : reportInfos) {
+            EmailZipFileDTO zipFileDTO = new EmailZipFileDTO(emailTitle, e);
             String reportPath = e.getReportPath();
             if (ArchiveUtil.isArchive(reportPath)) {
                 try {
                     this.handleCompressedFiles(emailTitle, reportPath, e.getReportType(), dtos);
                 } catch (Exception ex) {
                     log.warn("报告{} 压缩包解压失败：{}", reportPath, ExceptionUtil.stacktraceToString(ex));
-                    ReportData reportData = new ReportData.DefaultReportData();
-                    dataList.add(new ParseResult<>(ReportParseStatus.ARCHIVE_FAIL, reportData));
+                    zipFileDTO.setParseStatus(0);
+                    zipFileDTO.setFailReason("压缩包解压失败");
+                    dtos.add(zipFileDTO);
                 }
             } else {
-                dtos.add(new EmailZipFileDTO(emailTitle, reportPath, e.getReportType()));
+                dtos.add(zipFileDTO);
             }
         }
         // 重新判断类型
         this.recheckEmailType(emailTitle, dtos);
+        // 解析数据和报告列表构建
         EmailInfoDTO emailInfo = new EmailInfoDTO(emailTitle, dtos);
+        if (StrUtil.isNotBlank(params.getContent())) {
+            emailInfo.setEmailContent(params.getContent());
+        }
         Long totalSize = dtos.stream().map(EmailZipFileDTO::getFileSize).reduce(0L, Long::sum);
         EmailParseInfoDO emailDo = this.buildEmailParseInfo("upload", emailInfo, totalSize);
         Integer emailId = this.saveEmailParseInfo(emailDo);
         for (EmailZipFileDTO zipFile : dtos) {
-            EmailFileInfoDO emailFile = this.saveEmailFileInfo(emailId, zipFile.getFilename(), zipFile.getFilepath());
+            EmailFileInfoDO emailFile = this.saveEmailFileInfo(emailId, zipFile);
             zipFile.setFileId(emailFile.getId());
         }
         // 解析并处理解析结果
+        List<ParseResult<ReportData>> dataList = ListUtil.list(false);
         this.parseAndUpdateResult(emailId, emailInfo, dataList);
         // 解析结果转换
         List<UploadReportResult> resultList = ListUtil.list(false);
@@ -395,7 +405,9 @@ public class EmailParseService {
         return resultList;
     }
 
-    private void parseAndUpdateResult(Integer emailId, EmailInfoDTO emailInfo, List<ParseResult<ReportData>> dataList) {
+    private void parseAndUpdateResult(Integer emailId,
+                                      EmailInfoDTO emailInfo,
+                                      List<ParseResult<ReportData>> dataList) {
         this.parseResults(emailInfo, dataList);
         String failReason = null;
         int emailParseStatus = EmailParseStatusConst.SUCCESS;
@@ -420,7 +432,8 @@ public class EmailParseService {
      */
     private void recheckEmailType(String emailTitle, List<EmailZipFileDTO> dtos) {
         for (EmailZipFileDTO emailFile : dtos) {
-            if (EmailTypeConst.SUPPORT_NO_OTHER_TYPES.contains(emailFile.getEmailType())) {
+            if (EmailTypeConst.SUPPORT_NO_OTHER_TYPES.contains(emailFile.getEmailType())
+                    || !Objects.equals(1, emailFile.getParseStatus())) {
                 continue;
             }
             Integer type = EmailUtil.getEmailTypeBySubject(emailTitle + emailFile.getFilename());
@@ -442,6 +455,8 @@ public class EmailParseService {
      * @param dtos       邮件信息（包含所有解压后的文件）
      */
     private void checkEmailFileInfo(String emailTitle, List<EmailZipFileDTO> dtos) {
+        // 删除前置处理失败的文件
+        dtos.removeIf(e -> !Objects.equals(1, e.getParseStatus()));
         // 如果压缩包里面既有pdf又有其他格式的文件，说明其他格式的文件是不需要解析的
         List<String> exts = dtos.stream().map(EmailZipFileDTO::getExtName).distinct().toList();
         if (exts.contains(Constants.FILE_PDF) && exts.size() > 1) {
@@ -460,7 +475,7 @@ public class EmailParseService {
             // 不支持的类型
             Integer type = dto.getEmailType();
             if (!EmailTypeConst.SUPPORT_EMAIL_TYPES.contains(type)) {
-                log.info("邮件{} 类型{} 不支持解析。", emailTitle, type);
+                log.warn("邮件{} 报告{} 的类型是{} 不支持解析。", emailTitle, filename, type);
                 removeIterator.remove();
             }
         }
@@ -491,7 +506,7 @@ public class EmailParseService {
             }
         }
         if (CollUtil.isEmpty(dtos)) {
-            log.info("邮件{} 所有文件都已经解析成功过，不能重复解析了", emailTitle);
+            log.info("邮件{} 经校验没有需要解析的报告文件。", emailTitle);
             return;
         }
         if (log.isInfoEnabled()) {
@@ -508,6 +523,7 @@ public class EmailParseService {
     private void parseResults(EmailInfoDTO emailInfo,
                               List<ParseResult<ReportData>> resultList) {
         String emailTitle = emailInfo.getEmailTitle();
+        // 拷贝一个新的集合，方便操作不影响原集合
         List<EmailZipFileDTO> dtos = ListUtil.toList(emailInfo.getEmailFileList());
         if (CollUtil.isEmpty(dtos)) {
             return;
@@ -540,42 +556,42 @@ public class EmailParseService {
                                                                EmailZipFileDTO zipFile) {
         Integer fileId = zipFile.getFileId();
         Integer emailType = zipFile.getEmailType();
-        String fileName = zipFile.getFilename();
+        String reportName = zipFile.getFilename();
         String filepath = zipFile.getFilepath();
         ParseResult<ReportData> result = new ParseResult<>();
         boolean reportFlag = emailType == null || !EmailTypeConst.SUPPORT_EMAIL_TYPES.contains(emailType);
-        if (reportFlag || StrUtil.isBlank(fileName) || fileName.endsWith(Constants.FILE_HTML)) {
-            return new ParseResult<>(ReportParseStatus.NOT_A_REPORT, null, fileName);
+        if (reportFlag || StrUtil.isBlank(reportName) || reportName.endsWith(Constants.FILE_HTML)) {
+            return new ParseResult<>(ReportParseStatus.NOT_A_REPORT, null, reportName);
         }
         // 类型识别---先识别季度报告，没有季度再识别年度报告，最后识别月报
-        ReportType reportType = ReportParseUtils.matchReportType(emailType, fileName);
+        ReportType reportType = ReportParseUtils.matchReportType(emailType, reportName);
         if (reportType == null) {
             reportType = ReportParseUtils.matchReportType(emailType, emailTitle);
             if (log.isDebugEnabled()) {
-                log.debug("报告{} 根据邮件主题{} 重新识别的类型是：{}", fileName, emailTitle, reportType);
+                log.debug("报告{} 根据邮件主题{} 重新识别的类型是：{}", reportName, emailTitle, reportType);
             }
         }
         // 解析器--根据文件后缀获取对应解析器，解析不了就用AI来解析
         ReportParserFileType fileType = ReportParserFileType.getBySuffix(zipFile.getExtName());
         // 不支持的格式
         if (fileType == null) {
-            ReportData reportData = this.buildNvlReportData(fileId, reportType, null, fileName);
-            return new ParseResult<>(ReportParseStatus.NO_SUPPORT_TEMPLATE, reportData, fileName);
+            ReportData reportData = this.buildNvlReportData(fileId, reportType, reportName);
+            return new ParseResult<>(ReportParseStatus.NO_SUPPORT_TEMPLATE, reportData, reportName);
         }
         // 不是定期报告的判断逻辑放在不支持的格式下面
         if (reportType == null) {
-            ReportData reportData = this.buildNvlReportData(fileId, ReportType.OTHER, null, fileName);
-            return new ParseResult<>(ReportParseStatus.NOT_A_REPORT, reportData, fileName);
+            ReportData reportData = this.buildNvlReportData(fileId, ReportType.OTHER, reportName);
+            return new ParseResult<>(ReportParseStatus.NOT_A_REPORT, reportData, reportName);
         }
 
         // docx转pdf
         if (Objects.equals(ReportParserFileType.WORD, fileType)) {
             try {
-                String outputFile = FileUtil.getParent(filepath, 1) + File.separator + FileUtil.mainName(fileName) + ".pdf";
+                String outputFile = FileUtil.getParent(filepath, 1) + File.separator + FileUtil.mainName(reportName) + ".pdf";
                 PdfUtil.convertDocxToPdf(filepath, outputFile);
                 filepath = outputFile;
             } catch (Exception e) {
-                log.warn("报告{} 转换为pdf失败：{}", fileName, ExceptionUtil.stacktraceToString(e));
+                log.warn("报告{} 转换为pdf失败：{}", reportName, ExceptionUtil.stacktraceToString(e));
             }
         }
         // 首页和尾页转为png图片，首页用来识别基金名称和基金代码、尾页用来识别印章和联系人
@@ -586,24 +602,24 @@ public class EmailParseService {
                 File outputFile = FileUtil.file(FileUtil.getParent(output, 1));
                 images = PdfUtil.convertFirstAndLastPagesToPng(filepath, outputFile, 300, zipFile.getPdfPwd());
                 if (log.isDebugEnabled()) {
-                    log.debug("报告{} 生成的图片地址是：\n{}", fileName, images);
+                    log.debug("报告{} 生成的图片地址是：\n{}", reportName, images);
                 }
             } catch (Exception e) {
-                log.warn("报告{} 生成图片失败：{}", fileName, ExceptionUtil.stacktraceToString(e));
+                log.warn("报告{} 生成图片失败：{}", reportName, ExceptionUtil.stacktraceToString(e));
             }
         } else if (Objects.equals(ReportParserFileType.IMG, fileType)) {
             try {
                 String outputFile = PdfUtil.compressAndSave(filepath);
                 images.add(outputFile);
             } catch (IOException e) {
-                log.error("报告{} 图片压缩失败，{}", fileName, ExceptionUtil.stacktraceToString(e));
+                log.error("报告{} 图片压缩失败，{}", reportName, ExceptionUtil.stacktraceToString(e));
             }
         }
 
         // ocr识别月报是否管理人版或协会版
         ReportMonthlyType monthlyType = ReportMonthlyType.NO_NEED;
         if (ReportType.MONTHLY == reportType) {
-            monthlyType = this.determineReportType(emailTitle, fileName, filepath, images);
+            monthlyType = this.determineReportType(emailTitle, reportName, filepath, images);
         }
         boolean isAmac = reportType == ReportType.ANNUALLY || reportType == ReportType.QUARTERLY
                 || (reportType == ReportType.MONTHLY && ReportMonthlyType.AMAC == monthlyType);
@@ -611,7 +627,7 @@ public class EmailParseService {
         boolean notSupportFile = false;
         // 解析报告
         ReportData reportData = null;
-        ReportParserParams params = new ReportParserParams(fileId, fileName, filepath, reportType);
+        ReportParserParams params = new ReportParserParams(fileId, reportName, filepath, reportType);
         long start = System.currentTimeMillis();
         try {
             if (isAmac || reportType == ReportType.LETTER) {
@@ -620,7 +636,7 @@ public class EmailParseService {
                 result = new ParseResult<>(1, "报告解析成功", reportData);
             }
         } catch (ReportParseException e) {
-            result = new ParseResult<>(e.getCode(), StrUtil.format(e.getMsg(), fileName), null);
+            result = new ParseResult<>(e.getCode(), StrUtil.format(e.getMsg(), reportName), null);
             log.warn("解析失败：{}", result.getMsg());
             if (e instanceof NotSupportReportException) {
                 notSupportFile = true;
@@ -632,18 +648,18 @@ public class EmailParseService {
             // 如果解析结果是空的就用AI工具解析一次
             if (reportData == null && !notSupportFile) {
                 if (log.isInfoEnabled()) {
-                    log.info("报告{} 是周报或管理人月报或其他类型或解析失败，用AI解析器解析", fileName);
+                    log.info("报告{} 是周报或管理人月报或其他类型或解析失败，用AI解析器解析", reportName);
                 }
                 try {
                     if (!isAmac && CollUtil.isNotEmpty(images)) {
                         filepath = images.get(0);
                     }
-                    params = new ReportParserParams(fileId, fileName, filepath, reportType);
+                    params = new ReportParserParams(fileId, reportName, filepath, reportType);
                     ReportParser<ReportData> instance = this.reportParserFactory.getInstance(reportType, ReportParserFileType.AI);
                     reportData = instance.parse(params);
                     result = new ParseResult<>(1, "报告解析成功--AI", reportData);
                 } catch (ReportParseException e) {
-                    result = new ParseResult<>(e.getCode(), StrUtil.format(e.getMsg(), fileName), null);
+                    result = new ParseResult<>(e.getCode(), StrUtil.format(e.getMsg(), reportName), null);
                     log.warn("AI解析失败：{}", result.getMsg());
                 } catch (Exception e) {
                     log.warn("AI解析错误：{}", ExceptionUtil.stacktraceToString(e));
@@ -651,7 +667,9 @@ public class EmailParseService {
                 }
             }
             // 当reportData==null时重新构建一个reportData对象
-            reportData = this.buildNvlReportData(fileId, reportType, reportData, fileName);
+            if (reportData == null) {
+                reportData = this.buildNvlReportData(fileId, reportType, reportName);
+            }
             if (reportData.getBaseInfo() != null) {
                 // 设置月报类型
                 reportData.getBaseInfo().setMonthlyType(monthlyType.getType());
@@ -661,14 +679,14 @@ public class EmailParseService {
                 }
             }
             // ocr信息提取（印章、联系人、基金名称和产品代码）
-            reportData = this.ocrReportData(fileId, reportType, monthlyType, reportData, fileName, senderEmail, images);
+            reportData = this.ocrReportData(fileId, reportType, monthlyType, reportData, reportName, senderEmail, images);
             result.setData(reportData);
             if (log.isInfoEnabled()) {
-                log.info("报告{} 解析耗时{}ms，结果是：{}", fileName, (System.currentTimeMillis() - start), reportData);
+                log.info("报告{} 解析耗时{}ms，结果是：{}", reportName, (System.currentTimeMillis() - start), reportData);
             }
         }
         // 保存报告解析结果
-        this.saveReportData(reportData, reportType, fileName);
+        this.saveReportData(reportData, reportType, reportName);
         return result;
     }
 
@@ -852,24 +870,18 @@ public class EmailParseService {
      *
      * @param fileId     文件ID
      * @param reportType 报告类型
-     * @param reportData 解析结果对象
-     * @param fileName   报告名称
+     * @param reportName 报告名称
      * @return /
      */
-    private ReportData buildNvlReportData(Integer fileId,
-                                          ReportType reportType,
-                                          ReportData reportData,
-                                          String fileName) {
-        if (reportData != null) {
-            return reportData;
-        }
+    private ReportData buildNvlReportData(Integer fileId, ReportType reportType, String reportName) {
+        ReportData reportData = null;
         if (reportType == null) {
             reportType = ReportType.OTHER;
         }
         ReportBaseInfoDTO baseInfo = new ReportBaseInfoDTO(fileId);
-        baseInfo.setReportName(fileName);
+        baseInfo.setReportName(reportName);
         baseInfo.setReportType(reportType.name());
-        String reportDate = ReportParseUtils.matchReportDate(reportType, fileName);
+        String reportDate = ReportParseUtils.matchReportDate(reportType, reportName);
         baseInfo.setReportDate(ConvertUtil.toDate(reportDate));
         ReportFundInfoDTO fundInfo = new ReportFundInfoDTO(fileId);
         if (ReportType.ANNUALLY == reportType) {
@@ -909,14 +921,19 @@ public class EmailParseService {
         } finally {
             writeWatch.stop();
             if (log.isInfoEnabled()) {
-                log.info("报告{}解析结果保存完成，耗时{}ms", fileName, writeWatch.getTotalTimeMillis());
+                log.info("报告{} 解析结果保存完成，耗时{}ms", fileName, writeWatch.getTotalTimeMillis());
             }
         }
     }
 
-    private EmailFileInfoDO saveEmailFileInfo(Integer emailId, String fileName, String filePath) {
-        EmailFileInfoDO emailFileInfoDO = buildEmailFileInfoDO(emailId, fileName, filePath);
-//        emailFileInfoDO.setAiFileId(null);
+    private EmailFileInfoDO saveEmailFileInfo(Integer emailId, EmailZipFileDTO zipFile) {
+        String filename = zipFile.getFilename();
+        String filepath = zipFile.getFilepath();
+        EmailFileInfoDO emailFileInfoDO = this.buildEmailFileInfoDO(emailId, filename, filepath);
+        if (!Objects.equals(1, zipFile.getParseStatus()) || StrUtil.isNotBlank(zipFile.getFailReason())) {
+            emailFileInfoDO.setParseStatus(zipFile.getParseStatus());
+            emailFileInfoDO.setFailReason(zipFile.getFailReason());
+        }
         if (emailFileInfoDO.getId() != null) {
             this.emailFileInfoMapper.updateTimeById(emailFileInfoDO.getId(), new Date());
             return emailFileInfoDO;
@@ -924,8 +941,8 @@ public class EmailParseService {
 
         LambdaQueryWrapper<EmailFileInfoDO> wrapper = Wrappers.lambdaQuery(EmailFileInfoDO.class)
                 .eq(EmailFileInfoDO::getEmailId, emailId)
-                .eq(EmailFileInfoDO::getFileName, fileName)
-                .eq(EmailFileInfoDO::getFilePath, filePath);
+                .eq(EmailFileInfoDO::getFileName, filename)
+                .eq(EmailFileInfoDO::getFilePath, filepath);
         List<EmailFileInfoDO> tempList = this.emailFileInfoMapper.selectList(wrapper);
         if (CollUtil.isNotEmpty(tempList)) {
             return tempList.get(0);
@@ -978,9 +995,10 @@ public class EmailParseService {
         emailParseInfoDO.setId(null);
         emailParseInfoDO.setSenderEmail(emailInfo.getSenderEmail());
         emailParseInfoDO.setEmail(emailAddress);
-        emailParseInfoDO.setEmailDate(DateUtil.parse(emailInfo.getEmailDate(), DateConst.YYYY_MM_DD_HH_MM_SS));
+        emailParseInfoDO.setEmailDate(DateUtil.parseDateTime(emailInfo.getEmailDate()));
         emailParseInfoDO.setParseDate(new Date());
         emailParseInfoDO.setEmailTitle(emailInfo.getEmailTitle());
+        emailParseInfoDO.setEmailContent(emailInfo.getEmailContent());
         emailParseInfoDO.setParseStatus(EmailParseStatusConst.SUCCESS);
         emailParseInfoDO.setAttrSize(totalSize);
         emailParseInfoDO.setIsvalid(1);
@@ -1062,7 +1080,7 @@ public class EmailParseService {
             }
             try {
                 Date emailDate = message.getSentDate();
-                String emailDateStr = DateUtil.format(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS);
+                String emailDateStr = DateUtil.formatDateTime(emailDate);
                 if (log.isInfoEnabled()) {
                     log.info("{} 邮件{} 数据获取中，邮件时间：{}", folderName, emailTitle, emailDateStr);
                 }
@@ -1178,16 +1196,16 @@ public class EmailParseService {
         emailContentInfoDTO.setFilePath(saveFile.getAbsolutePath());
         emailContentInfoDTO.setEmailAddress(account);
         emailContentInfoDTO.setEmailTitle(subject);
-        emailContentInfoDTO.setEmailDate(DateUtil.format(sendDate, DateConst.YYYY_MM_DD_HH_MM_SS));
+        emailContentInfoDTO.setEmailDate(DateUtil.formatDateTime(sendDate));
         emailContentInfoDTOList.add(emailContentInfoDTO);
     }
 
     public File generateSavePath(String account, Date sendDate, String fileName) {
-        String emailDateStr = DateUtil.format(sendDate, DateConst.YYYYMMDD);
+        String emailDateStr = DateUtil.formatDate(sendDate);
         String filePath = this.path + File.separator + account + File.separator +
                 emailDateStr + File.separator + "original" + File.separator;
         // 压缩包重名时的后面的压缩包会覆盖前面压缩包的问题（不考虑普通文件）
-        String emailDate = DateUtil.format(sendDate, DateConst.YYYYMMDDHHMMSS24);
+        String emailDate = DateUtil.format(sendDate, DatePattern.PURE_DATETIME_PATTERN);
         String realName = ArchiveUtil.isArchive(fileName) ? emailDate + fileName : fileName;
         return FileUtil.file(filePath + realName);
     }
@@ -1202,7 +1220,7 @@ public class EmailParseService {
                 if (log.isDebugEnabled()) {
                     log.debug("邮件{} 获取的正文不做解析，内容是 {}", subject, bodyPartContent);
                 }
-                if (StrUtil.startWithIgnoreCase(bodyPart.getContentType(), MediaType.TEXT_PLAIN_VALUE)) {
+                if (StrUtil.startWithIgnoreCase(bodyPart.getContentType(), MediaType.TEXT_HTML_VALUE)) {
                     contents[0] = bodyPartContent.toString();
                 }
                 continue;

mo-daq/src/main/java/com/smppw/modaq/infrastructure/util/ConvertUtil.java

@@ -3,7 +3,6 @@ package com.smppw.modaq.infrastructure.util;
 import cn.hutool.core.date.DatePattern;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.util.StrUtil;
-import com.smppw.modaq.common.conts.DateConst;
 import com.smppw.modaq.common.conts.PatternConsts;
 
 import java.math.BigDecimal;
@@ -33,8 +32,9 @@ public class ConvertUtil {
                 // 日期格式化，支持格式如下：yyyy年MM月dd日、yyyy-MM-dd、yyyy/MM/dd、yyyy_MM_dd和yyyyMMdd
                 return DateUtil.parse(matcher.group(),
                         DatePattern.CHINESE_DATE_PATTERN,
-                        DateConst.YYYYMMDD,
+                        DatePattern.PURE_DATE_PATTERN,
                         DatePattern.NORM_DATE_PATTERN,
+                        DatePattern.CHINESE_DATE_PATTERN,
                         "yyyy/MM/dd",
                         "yyyy_MM_dd",
                         "yyyy.MM.dd",

mo-daq/src/main/resources/mapper/EmailFileInfoMapper.xml

@@ -7,6 +7,8 @@
         <result column="file_name" property="fileName"/>
         <result column="file_path" property="filePath"/>
         <result column="ai_parse" property="aiParse"/>
+        <result column="parse_status" property="parseStatus"/>
+        <result column="fail_reason" property="failReason"/>
 <!--        <result column="ai_file_id" property="aiFileId"/>-->
         <result column="isvalid" property="isvalid"/>
         <result column="creatorid" property="creatorId"/>
@@ -17,9 +19,9 @@
 
     <insert id="insertById" parameterType="com.smppw.modaq.domain.entity.EmailFileInfoDO" useGeneratedKeys="true"
             keyProperty="id" keyColumn="id">
-        insert into mo_email_file_info(email_id, file_name, file_path,
+        insert into mo_email_file_info(email_id, file_name, file_path,parse_status,fail_reason,
                                               isvalid, creatorid, createtime, updaterid, updatetime)
-        values (#{itemDo.emailId}, #{itemDo.fileName}, #{itemDo.filePath},
+        values (#{itemDo.emailId}, #{itemDo.fileName}, #{itemDo.filePath},#{itemDo.parseStatus},#{itemDo.failReason},
                 #{itemDo.isvalid}, #{itemDo.creatorId}, #{itemDo.createTime}, #{itemDo.updaterId}, #{itemDo.updateTime})
     </insert>
 
@@ -37,12 +39,12 @@
 <!--        from mo_email_file_info where email_id=#{emailId} and isvalid=1-->
 <!--    </select>-->
 
-    <select id="queryByEmailId" resultMap="BaseResultMap">
-        select id, file_name, file_path,ai_parse
-        from mo_email_file_info
-        where email_id = #{emailId}
-          and isvalid = 1
-    </select>
+<!--    <select id="queryByEmailId" resultMap="BaseResultMap">-->
+<!--        select id, file_name, file_path,ai_parse-->
+<!--        from mo_email_file_info-->
+<!--        where email_id = #{emailId}-->
+<!--          and isvalid = 1-->
+<!--    </select>-->
 <!--    <select id="searchFundFileInfo" resultType="com.simuwang.base.pojo.vo.FundFileInfoVO">-->
 <!--        SELECT DISTINCT efi.id as "fileId",-->
 <!--        <if test="fileType != 3">-->
@@ -226,14 +228,14 @@
           and id = #{id}
     </update>
 
-    <update id="updateAiParseByFileId">
-        update mo_email_file_info
-        set updatetime = now(),
-            ai_parse = #{aiParse},
-            ai_file_id = #{aiFileId}
-        where isvalid = 1
-          and id = #{fileId}
-    </update>
+<!--    <update id="updateAiParseByFileId">-->
+<!--        update mo_email_file_info-->
+<!--        set updatetime = now(),-->
+<!--            ai_parse = #{aiParse},-->
+<!--            ai_file_id = #{aiFileId}-->
+<!--        where isvalid = 1-->
+<!--          and id = #{fileId}-->
+<!--    </update>-->
 
     <update id="batchUpdateByFileId">
         <foreach collection="entityList" item="entity">
@@ -253,7 +255,7 @@
                  join mo_report_base_info b on b.file_id = a.id and b.report_type = 'LETTER'
                  join mo_report_fund_transaction c on a.id = c.file_id and c.fund_name is not null
                  join mo_report_investor_info d on a.id = d.file_id and d.investor_name is not null
-        where a.file_name = #{filename} and a.isvalid = 1
+        where a.file_name = #{filename} and a.isvalid = 1 and a.parse_status = 1
     </select>
 
     <select id="getAmacFilenameSuccessCount" resultType="int">
@@ -262,7 +264,7 @@
                  join mo_email_parse_info e on e.id = a.email_id and e.email_title = #{emailTitle} and e.attr_size = #{fileSize}
                  join mo_report_base_info b on b.file_id = a.id and b.report_type in ('MONTHLY', 'QUARTERLY', 'ANNUALLY')
                  join mo_report_fund_info c on a.id = c.file_id and (c.fund_name is not null or c.fund_code is not null)
-        where a.file_name = #{filename} and a.isvalid = 1
+        where a.file_name = #{filename} and a.isvalid = 1 and a.parse_status = 1
     </select>
 
     <select id="getWeeklyFilenameSuccessCount" resultType="int">
@@ -271,7 +273,7 @@
                  join mo_email_parse_info e on e.id = a.email_id and e.email_title = #{emailTitle} and e.attr_size = #{fileSize}
                  join mo_report_base_info b on b.file_id = a.id and b.report_type = 'WEEKLY'
                  join mo_report_fund_info c on a.id = c.file_id and (c.fund_name is not null or c.fund_code is not null)
-        where a.file_name = #{filename} and a.isvalid = 1
+        where a.file_name = #{filename} and a.isvalid = 1 and a.parse_status = 1
     </select>
 
     <select id="getOtherFilenameSuccessCount" resultType="int">
@@ -279,6 +281,6 @@
         from mo_email_file_info a
                  join mo_email_parse_info e on e.id = a.email_id and e.email_title = #{emailTitle} and e.attr_size = #{fileSize}
                  join mo_report_base_info b on b.file_id = a.id and b.report_type = 'OTHER'
-        where a.file_name = #{filename} and a.isvalid = 1
+        where a.file_name = #{filename} and a.isvalid = 1 and a.parse_status = 1
     </select>
 </mapper>

mo-daq/src/test/java/com/smppw/modaq/MoDaqApplicationTests.java

@@ -2,8 +2,6 @@ package com.smppw.modaq;
 
 import cn.hutool.core.collection.ListUtil;
 import cn.hutool.core.date.DateUtil;
-import com.smppw.modaq.application.service.EmailParseApiService;
-import com.smppw.modaq.common.conts.DateConst;
 import com.smppw.modaq.common.conts.EmailTypeConst;
 import com.smppw.modaq.domain.dto.MailboxInfoDTO;
 import com.smppw.modaq.domain.service.EmailParseService;
@@ -20,14 +18,11 @@ public class MoDaqApplicationTests {
     @Autowired
     private EmailParseService emailParseService;
 
-    @Autowired
-    private EmailParseApiService emailParseApiService;
-
     @Test
     public void letterTest() {
         MailboxInfoDTO emailInfoDTO = this.buildMailbox("**@simuwang.com", "**");
-        Date startDate = DateUtil.parse("2025-06-18 17:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2025-06-18 18:56:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parseDateTime("2025-06-18 17:00:00");
+        Date endDate = DateUtil.parseDateTime("2025-06-18 18:56:00");
         try {
             List<String> folderNames = ListUtil.list(false);
 //            folderNames.add("其他文件夹/报告公告");
@@ -42,8 +37,8 @@ public class MoDaqApplicationTests {
     @Test
     public void reportTest() {
         MailboxInfoDTO emailInfoDTO = this.buildMailbox("*@simuwang.com", "*");
-        Date startDate = DateUtil.parse("2025-06-19 10:30:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2025-06-19 17:57:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parseDateTime("2025-06-19 17:30:00");
+        Date endDate = DateUtil.parseDateTime("2025-06-19 17:57:00");
         try {
             List<String> folderNames = ListUtil.list(false);
 //            folderNames.add("其他文件夹/报告公告");