feat:邮件解析-增加邮件解析失败的原因(估值表和净值文件)

service-base/src/main/java/com/simuwang/base/common/util/NavDataUtil.java

@@ -39,4 +39,26 @@ public class NavDataUtil {
         log.info("数据格式验证通过 -> {}", fundNavDTO);
         return true;
     }
+
+    public static String checkDataFailReason(EmailFundNavDTO fundNavDTO) {
+        // 净值日期格式校验
+        if (StrUtil.isBlank(fundNavDTO.getPriceDate())) {
+            return "净值日期为空";
+        }
+        if (StrUtil.isBlank(fundNavDTO.getFundName()) && StrUtil.isBlank(fundNavDTO.getRegisterNumber())) {
+            return "备案编码和基金名称都为空";
+        }
+        if(StrUtil.isBlank(fundNavDTO.getNav()) && StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal()) && StrUtil.isBlank(fundNavDTO.getAssetNet())){
+            return "单位净值、累计净值、资产净值都为空";
+        }
+        // 单位净值,累计单位净值,资产净值,资产份额数字格式校验
+        boolean isvalidNumericFormat = (StrUtil.isBlank(fundNavDTO.getNav()) || StringUtil.isNumeric(fundNavDTO.getNav()))
+                && (StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal()) || StringUtil.isNumeric(fundNavDTO.getCumulativeNavWithdrawal()))
+                && (StrUtil.isBlank(fundNavDTO.getAssetNet()) || StringUtil.isNumeric(fundNavDTO.getAssetNet()))
+                && (StrUtil.isBlank(fundNavDTO.getAssetShare()) || StringUtil.isNumeric(fundNavDTO.getAssetShare()));
+        if (!isvalidNumericFormat) {
+            return "单位净值或累计净值或资产净值或资产份额格式不正确";
+        }
+        return null;
+    }
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailFieldMappingMapper.java

@@ -13,6 +13,6 @@ public interface EmailFieldMappingMapper {
      *
      * @return 净值文件字段识别映射配置
      */
-    List<EmailFieldMappingDO> getEmailFieldMapping();
+    List<EmailFieldMappingDO> getEmailFieldMapping(Integer type);
 
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailParseInfoMapper.java

@@ -15,7 +15,7 @@ public interface EmailParseInfoMapper {
 
     Integer insert(@Param("itemDo") EmailParseInfoDO emailParseInfoDO);
 
-    void updateParseStatus(@Param("id") Integer id, @Param("parseStatus") int parseStatus);
+    void updateParseStatus(@Param("id") Integer id, @Param("parseStatus") int parseStatus, @Param("failReason") String failReason);
 
     List<EmailParseInfoDO> searchEmailList(EmailParseQuery emailParseQuery);
 

service-base/src/main/java/com/simuwang/base/pojo/dos/EmailParseInfoDO.java

@@ -53,6 +53,11 @@ public class EmailParseInfoDO {
     @TableField(value = "parse_status")
     private Integer parseStatus;
     /**
+     * 失败原因
+     */
+    @TableField(value = "fail_reason")
+    private String failReason;
+    /**
      * 记录的有效性；1-有效；0-无效；
      */
     @TableField(value = "isvalid")

service-base/src/main/java/com/simuwang/base/pojo/dto/EmailFundNavDTO.java

@@ -67,6 +67,10 @@ public class EmailFundNavDTO {
      * 估值表原始信息
      */
     private List<CmValuationTableAttribute> valuationTableAttributeList;
+    /**
+     * 失败原因
+     */
+    private String failReason;
 
     @Override
     public boolean equals(Object o) {

service-base/src/main/resources/mapper/EmailFieldMappingMapper.xml

@@ -16,6 +16,9 @@
         select *
         from PPW_EMAIL.email_field_mapping
         where isvalid = 1
+        <if test="type != null">
+            and(TYPE =#{type} or TYPE = 0)
+        </if>
     </select>
 
 </mapper>

service-base/src/main/resources/mapper/EmailParseInfoMapper.xml

@@ -10,6 +10,7 @@
         <result column="email_title" property="emailTitle"/>
         <result column="email_type" property="emailType"/>
         <result column="parse_status" property="parseStatus"/>
+        <result column="fail_reason" property="failReason"/>
         <result column="isvalid" property="isvalid"/>
         <result column="creatorid" property="creatorId"/>
         <result column="createtime" property="createTime"/>
@@ -28,7 +29,8 @@
 
     <update id="updateParseStatus">
         update PPW_EMAIL.email_parse_info
-        set parse_status = #{parseStatus}
+        set parse_status = #{parseStatus},
+            fail_reason = #{failReason}
         where isvalid = 1
           and id = #{id}
     </update>

service-daq/src/main/java/com/simuwang/daq/components/PDMonthlyReportParser.java

@@ -77,7 +77,7 @@ public class PDMonthlyReportParser extends AbstractReportParser<MonthlyReportNav
                 }
             }
         }
-        List<EmailFieldMappingDO> emailFieldMapping = this.fieldMappingMapper.getEmailFieldMapping();
+        List<EmailFieldMappingDO> emailFieldMapping = this.fieldMappingMapper.getEmailFieldMapping(1);
         if (CollUtil.isNotEmpty(emailFieldMapping)) {
             this.fieldMapper = emailFieldMapping.stream().map(e -> new ValueLabelVO(e.getCode(), e.getName())).collect(Collectors.toList());
         }

service-daq/src/main/java/com/simuwang/daq/service/AbstractEmailParser.java

@@ -19,7 +19,7 @@ public abstract class AbstractEmailParser {
 
     public abstract List<EmailFundNavDTO> parse(EmailContentInfoDTO emailContentInfoDTO, Map<String, List<String>> emailFieldMap);
 
-    public boolean dataFormat(EmailFundNavDTO fundNavDTO) {
-        return NavDataUtil.navDataFormatCheck(fundNavDTO);
+    public String checkDataFailReason(EmailFundNavDTO fundNavDTO) {
+        return NavDataUtil.checkDataFailReason(fundNavDTO);
     }
 }

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -186,6 +186,10 @@ public class EmailParseService {
             Integer fileId = saveEmailFileInfo(emailId, emailContentInfoDTO.getFileId(), fileName, emailContentInfoDTO.getFilePath(), parseDate);
 
             List<EmailFundNavDTO> fundNavDTOList = fileNameNavEntry.getValue();
+            if(CollUtil.isNotEmpty(fundNavDTOList)){
+                // 过滤出解析成功的数据
+                fundNavDTOList = fundNavDTOList.stream().filter(e -> e != null && StrUtil.isBlank(e.getFailReason())).toList();
+            }
             if (CollUtil.isEmpty(fundNavDTOList) && !Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) {
                 continue;
             }
@@ -206,14 +210,20 @@ public class EmailParseService {
         }
 
         // 更新邮件解析结果 -> 当【净值日期】和【备案编码/基金名称】能正常解读，即识别为【成功】
-        long successNavCount = fileNameNavMap.values().stream().flatMap(List::stream).filter(Objects::nonNull).count();
+        long successNavCount = fileNameNavMap.values().stream().flatMap(List::stream).filter(e -> e != null && StrUtil.isBlank(e.getFailReason())).count();
         emailParseStatus = successNavCount >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
         // 报告邮件有一条成功就表示整体成功
         if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) && CollUtil.isNotEmpty(dataList)) {
             long count = dataList.size();
             emailParseStatus = count >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
         }
-        emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus);
+        String failReason = null;
+        if (emailParseStatus == EmailParseStatusConst.FAIL) {
+            // 邮件解析失败时 -> 保存失败原因
+            failReason = fileNameNavMap.values().stream().flatMap(List::stream).
+                    map(EmailFundNavDTO::getFailReason).distinct().collect(Collectors.joining("/"));
+        }
+        emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus, failReason);
     }
 
     private void saveValuationInfo(Integer fileId, List<EmailFundNavDTO> fundNavDTOList) {
@@ -730,7 +740,7 @@ public class EmailParseService {
     }
 
     public Map<String, List<String>> getEmailFieldMapping() {
-        List<EmailFieldMappingDO> emailFieldMappingDOList = emailFieldMapper.getEmailFieldMapping();
+        List<EmailFieldMappingDO> emailFieldMappingDOList = emailFieldMapper.getEmailFieldMapping(1);
         return emailFieldMappingDOList.stream()
                 .collect(Collectors.toMap(EmailFieldMappingDO::getCode, v -> Arrays.stream(v.getName().split(",")).toList()));
     }

service-daq/src/main/java/com/simuwang/daq/service/NavEmailParser.java

@@ -166,9 +166,9 @@ public class NavEmailParser extends AbstractEmailParser {
         }
         // 2.解析sheet中的净值数据
         List<EmailFundNavDTO> emailFundNavDTOList = parseSheetData(filePath, sheet, fieldPositionMap, null);
-        // 3.校验净值数据格式
+        // 3.校验净值数据格式 并 设置数据校验不通过的原因
         if (CollUtil.isNotEmpty(emailFundNavDTOList)) {
-            emailFundNavDTOList = emailFundNavDTOList.stream().filter(super::dataFormat).collect(Collectors.toList());
+            emailFundNavDTOList.forEach(e -> e.setFailReason(super.checkDataFailReason(e)));
         }
         return emailFundNavDTOList;
     }
@@ -316,9 +316,6 @@ public class NavEmailParser extends AbstractEmailParser {
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL))) : null;
         String assetNet = columnFieldMap.get(EmailFieldConst.ASSET_NET) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET)) != null ?
                 ExcelUtil.getCellValue(sheetRow.getCell(columnFieldMap.get(EmailFieldConst.ASSET_NET))) : null;
-        if (StrUtil.isBlank(nav) && StrUtil.isBlank(cumulativeNavWithdrawal) && StrUtil.isBlank(assetNet)) {
-            return null;
-        }
         List<EmailFundNavDTO> fundNavDTOList = CollUtil.newArrayList();
         EmailFundNavDTO emailFundNavDTO = new EmailFundNavDTO();
         String priceDate = columnFieldMap.get(EmailFieldConst.PRICE_DATE) != null && sheetRow.getCell(columnFieldMap.get(EmailFieldConst.PRICE_DATE)) != null ?

service-daq/src/main/java/com/simuwang/daq/service/ValuationEmailParser.java

@@ -50,15 +50,10 @@ public class ValuationEmailParser extends AbstractEmailParser {
         List<ValuationNeedParseParam> valuationNeedParseParams = buildValuationNeedParseParam(emailContentInfoDTO);
         List<AssetsValuationResult.Record> recordList = valuationParseService.parseValuationExcel(valuationNeedParseParams);
         if (CollUtil.isNotEmpty(recordList)) {
-            List<AssetsValuationResult.Record> parseSuccessList = recordList.stream()
-                    .filter(e -> e.getSuccess() == 1 || (StrUtil.isNotBlank(e.getMsg()) && "未匹配基金".equals(e.getMsg()))).collect(Collectors.toList());
-            List<EmailFundNavDTO> fundNavDTOList = convertToFundNavDTO(parseSuccessList);
+            List<EmailFundNavDTO> fundNavDTOList = convertToFundNavDTO(recordList);
             Optional.ofNullable(fundNavDTOList).ifPresent(emailFundNavDTOList::addAll);
         }
-        // 校验净值数据格式
-        if (CollUtil.isNotEmpty(emailFundNavDTOList)) {
-            emailFundNavDTOList = emailFundNavDTOList.stream().filter(super::dataFormat).collect(Collectors.toList());
-        }
+
         return emailFundNavDTOList;
     }
 
@@ -154,6 +149,7 @@ public class ValuationEmailParser extends AbstractEmailParser {
             fundNavDTO.setValuationTableDO(record.getValuationTableDO());
             fundNavDTO.setValuationTableAttributeList(record.getValuationTableAttributeList());
             fundNavDTO.setFundPositionDetailDOList(record.getFundPositionDetailDOList());
+            fundNavDTO.setFailReason(record.getMsg());
             emailFundNavDTOList.add(fundNavDTO);
         }
         return emailFundNavDTOList;

service-deploy/src/main/resources/application.yml

@@ -71,7 +71,7 @@ email:
   file:
     path: /home/wwwroot/shzq_dataapi/file/nav
   parse:
-    force-template-enable: true
+    force-template-enable: false
 
 # 配置
 simuwang:

service-deploy/src/test/java/com/simuwang/ApplicationTest.java

@@ -39,8 +39,8 @@ public class ApplicationTest {
 //        emailInfoDTO.setPort("993");
 //        emailInfoDTO.setProtocol("imap");
 
-        Date startDate = DateUtil.parse("2024-09-27 16:25:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-09-27 16:40:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-10-10 15:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2024-10-10 16:40:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {