feat:模板配置功能的实现

service-base/src/main/java/com/simuwang/base/common/conts/ApplicationRuleFileConst.java

@@ -1,16 +1,25 @@
 package com.simuwang.base.common.conts;
 
 public class ApplicationRuleFileConst {
+
     /**
      * 邮件标题
      */
     public final static Integer EMAIL_TITLE_FILE = 1;
+
+    /**
+     * 邮件正文
+     */
+    public final static Integer EMAIL_CONTENT_FILE = 2;
+
     /**
      * 附件名称
      */
-    public final static Integer EMAIL_FILE_NAME_FILE = 2;
+    public final static Integer EMAIL_FILE_NAME_FILE = 3;
+
     /**
      * 表格内容
      */
-    public final static Integer EMAIL_EXCEL_CONTENT_FILE = 3;
+    public final static Integer EMAIL_EXCEL_CONTENT_FILE = 4;
+
 }

service-base/src/main/java/com/simuwang/base/common/conts/EmailDataDirectionConst.java

@@ -0,0 +1,25 @@
+package com.simuwang.base.common.conts;
+
+public class EmailDataDirectionConst {
+
+    /**
+     * 数据行方向
+     */
+    public static final Integer ROW_DIRECTION_TYPE = 1;
+
+    /**
+     * 数据列方向
+     */
+    public static final Integer COLUMN_DIRECTION_TYPE = 2;
+
+    /**
+     * 其他方向
+     */
+    public static final Integer OTHER_DIRECTION_TYPE = 3;
+
+    /**
+     * 最大行数或最大列数
+     */
+    public static final Integer MAX_ROW_COLUMN = 2;
+
+}

service-base/src/main/java/com/simuwang/base/common/util/EmailUtil.java

@@ -161,7 +161,6 @@ public class EmailUtil {
         EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO();
         try {
             if ((part.getContentType().contains("text/html") || part.getContentType().contains("TEXT/HTML"))) {
-                if (partContent.toString().contains("<table")) {
                     emailContentInfoDTO.setEmailContent(partContent.toString());
                     String savePath = filePath + fileName;
                     File saveFile = new File(savePath);
@@ -174,7 +173,6 @@ public class EmailUtil {
                     }
                     emailContentInfoDTO.setFileName(fileName);
                     emailContentInfoDTO.setFilePath(savePath);
-                }
             }
         } catch (MessagingException e) {
             logger.info("邮件正文采集失败 -> 文件名:{}, 报错堆栈:{}", fileName, ExceptionUtil.stacktraceToString(e));

service-base/src/main/java/com/simuwang/base/common/util/ExcelUtil.java

@@ -3,17 +3,26 @@ package com.simuwang.base.common.util;
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.exceptions.ExceptionUtil;
+import cn.hutool.core.lang.Pair;
 import cn.hutool.core.util.StrUtil;
 import com.simuwang.base.common.conts.DateConst;
+import com.simuwang.base.common.conts.EmailDataDirectionConst;
+import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.poi.hssf.usermodel.*;
 import org.apache.poi.ss.usermodel.*;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.text.NumberFormat;
 import java.util.Date;
 import java.util.List;
@@ -23,6 +32,11 @@ import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.ArchiveInputStream;
 import org.apache.commons.compress.archivers.ArchiveStreamFactory;
 import org.apache.commons.compress.utils.IOUtils;
+import technology.tabula.ObjectExtractor;
+import technology.tabula.Page;
+import technology.tabula.PageIterator;
+import technology.tabula.RectangularTextContainer;
+import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
 
 public class ExcelUtil {
 
@@ -43,6 +57,7 @@ public class ExcelUtil {
     public static boolean isHTML(String fileName) {
         return StrUtil.isNotBlank(fileName) && fileName.endsWith("html");
     }
+
     public static Sheet getSheet(File file, int sheetIndex) {
         if (file == null || !file.exists()) {
             return null;
@@ -326,12 +341,97 @@ public class ExcelUtil {
     }
 
     public static Sheet getFirstSheet(String filePath) {
+        if (StrUtil.isBlank(filePath)) {
+            return null;
+        }
         try {
             File file = new File(filePath);
             return ExcelUtil.getSheet(file, 0);
-        }catch (Exception e) {
-            logger.error("获取文件的sheet错误 -> 文件路径:{},堆栈信息为：{}",filePath, ExceptionUtil.stacktraceToString(e));
+        } catch (Exception e) {
+            logger.error("获取文件的sheet错误 -> 文件路径:{},堆栈信息为：{}", filePath, ExceptionUtil.stacktraceToString(e));
         }
         return null;
     }
+
+    public static String pdfConvertToExcel(String filePath, String excelFilePath) {
+        File savefile = new File(excelFilePath);
+        if (!savefile.exists()) {
+            if (!savefile.getParentFile().exists()) {
+                savefile.getParentFile().mkdirs();
+                savefile.getParentFile().setExecutable(true);
+            }
+        }
+        try (OutputStream outputStream = Files.newOutputStream(Paths.get(excelFilePath))) {
+            PDDocument document = Loader.loadPDF(new File(filePath));
+            PageIterator extract = new ObjectExtractor(document).extract();
+            Workbook workbook = new XSSFWorkbook();
+            Sheet sheet = workbook.createSheet("Sheet1");
+            while (extract.hasNext()) {
+                Page next = extract.next();
+                List<technology.tabula.Table> tableList = new SpreadsheetExtractionAlgorithm().extract(next);
+                for (technology.tabula.Table table : tableList) {
+                    List<List<RectangularTextContainer>> rows = table.getRows();
+                    for (int rowNum = 0; rowNum < rows.size(); rowNum++) {
+                        Row sheetRow = sheet.createRow(rowNum);
+                        List<RectangularTextContainer> textContainerList = rows.get(rowNum);
+                        for (int cellNum = 0; cellNum < textContainerList.size(); cellNum++) {
+                            Cell cell = sheetRow.createCell(cellNum);
+                            RectangularTextContainer textContainer = textContainerList.get(cellNum);
+                            if (textContainer != null) {
+                                cell.setCellValue(textContainer.getText());
+                            }
+                        }
+                    }
+                }
+            }
+            // 将Excel工作簿写入输出流
+            workbook.write(outputStream);
+        } catch (Exception e) {
+            logger.error("解析邮件pdf附件报错 -> 堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
+            return null;
+        }
+        return excelFilePath;
+    }
+
+    public static String contentConvertToExcel(String emailContent, String excelFilePath) {
+        Elements rows;
+        try {
+            Document doc = Jsoup.parse(emailContent);
+            Element table = doc.select("table").first();
+            rows = table.select("tr");
+        } catch (Exception e) {
+            return null;
+        }
+        File saveFile = new File(excelFilePath);
+        if (!saveFile.exists()) {
+            if (!saveFile.getParentFile().exists()) {
+                saveFile.getParentFile().mkdirs();
+                saveFile.getParentFile().setExecutable(true);
+            }
+        }
+        try (OutputStream outputStream = new FileOutputStream(saveFile)) {
+            // 创建一个新的Excel工作簿
+            Workbook workbook = new XSSFWorkbook();
+            Sheet sheet = workbook.createSheet("Sheet1");
+            ExcelUtil.writeDataToSheet(sheet, rows);
+            // 将Excel工作簿写入输出流
+            workbook.write(outputStream);
+        } catch (Exception e) {
+            logger.error("邮件正文转换成excel报错 -> 堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
+            return null;
+        }
+        return excelFilePath;
+    }
+
+    /**
+     * 通过表头所在位置判断是行数据还是列数据
+     *
+     * @param fieldPositionMap excel中表头所在的位置
+     * @return 行方向-1,,列方向-2
+     */
+    public static Integer detectDataDirection(Map<String, Pair<Integer, Integer>> fieldPositionMap) {
+        long count = fieldPositionMap.values().stream().map(Pair::getValue).distinct().count();
+        return count == 1 ? EmailDataDirectionConst.COLUMN_DIRECTION_TYPE : EmailDataDirectionConst.ROW_DIRECTION_TYPE;
+    }
+
 }

service-base/src/main/java/com/simuwang/base/common/util/NavDataUtil.java

@@ -0,0 +1,35 @@
+package com.simuwang.base.common.util;
+
+import cn.hutool.core.util.StrUtil;
+import com.simuwang.base.pojo.dto.EmailFundNavDTO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NavDataUtil {
+
+    private static final Logger log = LoggerFactory.getLogger(NavDataUtil.class);
+
+    public static boolean navDataFormatCheck(EmailFundNavDTO fundNavDTO) {
+        // 净值日期格式校验
+        boolean isvalidDate = StringUtil.isValidDate(fundNavDTO.getPriceDate());
+        if (!isvalidDate) {
+            log.warn("净值日期格式不正确 -> 解析到的数据:{}", fundNavDTO);
+            return false;
+        }
+        if(StrUtil.isBlank(fundNavDTO.getFundName()) && StrUtil.isBlank(fundNavDTO.getRegisterNumber())){
+            log.warn("备案编码和基金名称都为空-> 解析到的数据:{}", fundNavDTO);
+            return false;
+        }
+        // 单位净值,累计单位净值,资产净值,资产份额数字格式校验
+        boolean isvalidNumericFormat = (StrUtil.isBlank(fundNavDTO.getNav()) || StringUtil.isNumeric(fundNavDTO.getNav()))
+                && (StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal()) || StringUtil.isNumeric(fundNavDTO.getCumulativeNavWithdrawal()))
+                && (StrUtil.isBlank(fundNavDTO.getAssetNet()) || StringUtil.isNumeric(fundNavDTO.getAssetNet()))
+                && (StrUtil.isBlank(fundNavDTO.getAssetShare()) || StringUtil.isNumeric(fundNavDTO.getAssetShare()));
+        if (!isvalidNumericFormat) {
+            log.warn("单位净值或累计净值或资产净值或资产份额格式不正确 -> 解析到的数据:{}", fundNavDTO);
+            return false;
+        }
+        log.info("数据格式验证通过 -> {}", fundNavDTO);
+        return true;
+    }
+}

service-base/src/main/java/com/simuwang/base/mapper/EmailTemplateMappingMapper.java

@@ -9,6 +9,6 @@ import java.util.List;
 @Mapper
 public interface EmailTemplateMappingMapper {
 
-    List<EmailTemplateInfoDO> queryByEmail(@Param("email") String email);
+    List<EmailTemplateInfoDO> queryByEmail(@Param("email") String email, @Param("type") Integer type);
 
 }

service-base/src/main/java/com/simuwang/base/pojo/EmailFileContentDTO.java

@@ -0,0 +1,21 @@
+package com.simuwang.base.pojo;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class EmailFileContentDTO {
+
+    /**
+     * 文件路径(excel路径)
+     */
+    private String filePath;
+
+    /**
+     * 邮件正文
+     */
+    private String content;
+}

service-base/src/main/java/com/simuwang/base/pojo/dos/EmailTemplateDataRuleDO.java

@@ -26,6 +26,11 @@ public class EmailTemplateDataRuleDO {
     @TableId(value = "field_name")
     private Integer fieldName;
     /**
+     * 提取位置：1-正文，2-正文表格
+     */
+    @TableId(value = "position")
+    private Integer position;
+    /**
      * 表格行
      */
     @TableId(value = "row")
@@ -39,7 +44,12 @@ public class EmailTemplateDataRuleDO {
      * 提取规则
      */
     @TableId(value = "field_rule")
-    private Integer fieldRule;
+    private String fieldRule;
+    /**
+     * 单位转换
+     */
+    @TableId(value = "unit_convert")
+    private Integer unitConvert;
     /**
      * 是否有效：0-无效，1-有效
      */

service-base/src/main/java/com/simuwang/base/pojo/dos/EmailTemplateInfoDO.java

@@ -21,11 +21,26 @@ public class EmailTemplateInfoDO {
     @TableId(value = "name")
     private String name;
     /**
+     * 模板类型：1-excel类型，2-正文类型
+     */
+    @TableId(value = "type")
+    private Integer type;
+    /**
      * 模版方向：1-行，2-列
      */
     @TableId(value = "direction")
     private Integer direction;
     /**
+     * 循环开始数字或者字母
+     */
+    @TableId(value = "start_index")
+    private String startIndex;
+    /**
+     * 循环结束数字或者字母
+     */
+    @TableId(value = "end_index")
+    private String endIndex;
+    /**
      * 模版备注
      */
     @TableId(value = "description")

service-base/src/main/java/com/simuwang/base/pojo/dto/EmailFundNavDTO.java

@@ -3,6 +3,7 @@ package com.simuwang.base.pojo.dto;
 import lombok.Data;
 
 import java.util.List;
+import java.util.Objects;
 
 @Data
 public class EmailFundNavDTO {
@@ -46,4 +47,22 @@ public class EmailFundNavDTO {
      * 解析状态:1-成功,2-净值缺失，3-未匹配基金，4-净值<=0,5-资产净值<=0
      */
     private Integer parseStatus;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        EmailFundNavDTO that = (EmailFundNavDTO) o;
+        return Objects.equals(fundName, that.fundName) && Objects.equals(registerNumber, that.registerNumber) && Objects.equals(priceDate, that.priceDate)
+                && Objects.equals(nav, that.nav) && Objects.equals(cumulativeNavWithdrawal, that.cumulativeNavWithdrawal) && Objects.equals(assetShare, that.assetShare) && Objects.equals(assetNet, that.assetNet);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(fundName, registerNumber, priceDate, nav, cumulativeNavWithdrawal, assetShare, assetNet);
+    }
 }

service-base/src/main/java/com/simuwang/base/pojo/dto/TemplateDataRuleDTO.java

@@ -0,0 +1,31 @@
+package com.simuwang.base.pojo.dto;
+
+import lombok.Data;
+
+@Data
+public class TemplateDataRuleDTO {
+    /**
+     * 字段名称：1-备案编码，2-基金名称，3-净值日期，4-单位净值，5-累计净值，6-资产份额，7-资产净值
+     */
+    private Integer fieldName;
+    /**
+     * 提取位置：1-正文，2-正文表格
+     */
+    private Integer position;
+    /**
+     * 表格行
+     */
+    private Integer row;
+    /**
+     * 表格列
+     */
+    private String column;
+    /**
+     * 提取规则
+     */
+    private String fieldRule;
+    /**
+     * 单位转换
+     */
+    private Integer unitConvert;
+}

service-base/src/main/java/com/simuwang/base/pojo/dto/TemplateDetailDTO.java

@@ -1,9 +1,8 @@
 package com.simuwang.base.pojo.dto;
 
-import cn.hutool.core.lang.Pair;
 import lombok.Data;
 
-import java.util.Map;
+import java.util.List;
 
 @Data
 public class TemplateDetailDTO {
@@ -18,7 +17,21 @@ public class TemplateDetailDTO {
     private Integer direction;
 
     /**
-     * 字段所在表格中的位置
+     * 模板类型：1-excel类型，2-正文类型
      */
-    private Map<String, Pair<Integer, Integer>> fieldPositionMap;
+    private Integer type;
+
+    /**
+     * 循环开始数字或者字母
+     */
+    private String startIndex;
+
+    /**
+     * 循环结束数字或者字母
+     */
+    private String endIndex;
+    /**
+     * 数据提取规则明细
+     */
+    private List<TemplateDataRuleDTO> dataRuleDetailList;
 }

service-base/src/main/resources/mapper/EmailParseInfoMapper.xml

@@ -114,7 +114,7 @@
     </select>
 
     <select id="queryById" resultMap="BaseResultMap">
-        select id, email, email_date, parse_date, email_title, email_type
+        select id, email, email_date, parse_date, email_title, email_type, sender_email
         from PPW_EMAIL.email_parse_info
         where isvalid = 1
           and id = #{id}

service-base/src/main/resources/mapper/EmailTemplateDataRuleMapper.xml

@@ -5,9 +5,11 @@
         <id column="id" property="id"/>
         <result column="template_id" property="templateId"/>
         <result column="field_name" property="fieldName"/>
+        <result column="position" property="position"/>
         <result column="row" property="row"/>
         <result column="column" property="column"/>
         <result column="field_rule" property="fieldRule"/>
+        <result column="unit_convert" property="unitConvert"/>
         <result column="isvalid" property="isvalid"/>
         <result column="creatorid" property="creatorId"/>
         <result column="createtime" property="createTime"/>

service-base/src/main/resources/mapper/EmailTemplateInfoMapper.xml

@@ -4,7 +4,10 @@
     <resultMap id="BaseResultMap" type="com.simuwang.base.pojo.dos.EmailTemplateInfoDO">
         <id column="id" property="id"/>
         <result column="name" property="name"/>
+        <result column="type" property="type"/>
         <result column="direction" property="direction"/>
+        <result column="start_index" property="startIndex"/>
+        <result column="end_index" property="endIndex"/>
         <result column="description" property="description"/>
         <result column="status" property="status"/>
         <result column="isvalid" property="isvalid"/>

service-base/src/main/resources/mapper/EmailTemplateMappingMapper.xml

@@ -18,10 +18,14 @@
 
     <select id="queryByEmail" resultType="com.simuwang.base.pojo.dos.EmailTemplateInfoDO">
         select t1.template_id as id,
-               t2.direction as direction
+               t2.direction as direction,
+               t2.type as type,
+               t2.start_index as startIndex,
+               t2.end_index as endIndex
         from PPW_EMAIL.email_template_mapping t1
-            join PPW_EMAIL.email_template_info t2 on t2.id = t1.template_id and t2.isvalid = 1
+            join PPW_EMAIL.email_template_info t2 on t2.id = t1.template_id and t2.isvalid = 1 and t2.status = 1
         where t1.email = #{email}
+          and t2.type = #{type}
           and t1.isvalid = 1
           and t2.status = 1
     </select>

service-daq/src/main/java/com/simuwang/daq/service/AbstractEmailParser.java

@@ -1,6 +1,7 @@
 package com.simuwang.daq.service;
 
 import cn.hutool.core.util.StrUtil;
+import com.simuwang.base.common.util.NavDataUtil;
 import com.simuwang.base.common.util.StringUtil;
 import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
 import com.simuwang.base.pojo.dto.EmailFundNavDTO;
@@ -12,8 +13,6 @@ import java.util.Map;
 
 public abstract class AbstractEmailParser {
 
-    private static final Logger log = LoggerFactory.getLogger(AbstractEmailParser.class);
-
     public boolean isSupport(Integer emailType) {
         return false;
     }
@@ -21,22 +20,6 @@ public abstract class AbstractEmailParser {
     public abstract List<EmailFundNavDTO> parse(EmailContentInfoDTO emailContentInfoDTO, Map<String, List<String>> emailFieldMap);
 
     public boolean dataFormat(EmailFundNavDTO fundNavDTO) {
-        // 净值日期格式校验
-        boolean isvalidDate = StringUtil.isValidDate(fundNavDTO.getPriceDate());
-        if (!isvalidDate) {
-            log.warn("净值日期格式不正确 -> 解析到的数据:{}", fundNavDTO);
-            return false;
-        }
-        // 单位净值,累计单位净值,资产净值,资产份额数字格式校验
-        boolean isvalidNumericFormat = (StrUtil.isBlank(fundNavDTO.getNav()) || StringUtil.isNumeric(fundNavDTO.getNav()))
-                && (StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal()) || StringUtil.isNumeric(fundNavDTO.getCumulativeNavWithdrawal()))
-                && (StrUtil.isBlank(fundNavDTO.getAssetNet()) || StringUtil.isNumeric(fundNavDTO.getAssetNet()))
-                && (StrUtil.isBlank(fundNavDTO.getAssetShare()) || StringUtil.isNumeric(fundNavDTO.getAssetShare()));
-        if (!isvalidNumericFormat) {
-            log.warn("单位净值或累计净值或资产净值或资产份额格式不正确 -> 解析到的数据:{}", fundNavDTO);
-            return false;
-        }
-        log.info("数据格式验证通过 -> {}", fundNavDTO);
-        return true;
+        return NavDataUtil.navDataFormatCheck(fundNavDTO);
     }
 }

service-daq/src/main/java/com/simuwang/daq/service/EmailParseApiServiceImpl.java

@@ -155,6 +155,7 @@ public class EmailParseApiServiceImpl implements EmailParseApiService {
             EmailContentInfoDTO contentInfoDTO = new EmailContentInfoDTO();
             contentInfoDTO.setEmailId(emailId);
             contentInfoDTO.setFileId(fileInfoDO.getId());
+            contentInfoDTO.setSenderEmail(emailParseInfoDO.getSenderEmail());
             contentInfoDTO.setEmailAddress(emailParseInfoDO.getEmail());
             contentInfoDTO.setEmailDate(emailDate);
             contentInfoDTO.setEmailTitle(emailParseInfoDO.getEmailTitle());

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -597,6 +597,7 @@ public class EmailParseService {
 
         // 正则表达式匹配邮件地址
         Pattern pattern = Pattern.compile("<(\\S+)>");
+
         Matcher matcher = pattern.matcher(address);
         if (matcher.find()) {
             return matcher.group(1);

service-daq/src/main/java/com/simuwang/daq/service/EmailTemplateService.java

@@ -1,33 +1,54 @@
 package com.simuwang.daq.service;
 
+import cn.hutool.core.bean.BeanUtil;
 import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.lang.Pair;
+import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.util.StrUtil;
 import com.simuwang.base.common.conts.ApplicationRuleFileConst;
+import com.simuwang.base.common.conts.EmailDataDirectionConst;
 import com.simuwang.base.common.conts.EmailFieldConst;
 import com.simuwang.base.common.util.ExcelUtil;
+import com.simuwang.base.common.util.NavDataUtil;
 import com.simuwang.base.mapper.EmailTemplateApplicationRuleMapper;
 import com.simuwang.base.mapper.EmailTemplateDataRuleMapper;
 import com.simuwang.base.mapper.EmailTemplateMappingMapper;
+import com.simuwang.base.pojo.EmailFileContentDTO;
 import com.simuwang.base.pojo.dos.EmailTemplateApplicationRuleDO;
 import com.simuwang.base.pojo.dos.EmailTemplateDataRuleDO;
 import com.simuwang.base.pojo.dos.EmailTemplateInfoDO;
 import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
+import com.simuwang.base.pojo.dto.EmailFundNavDTO;
+import com.simuwang.base.pojo.dto.TemplateDataRuleDTO;
 import com.simuwang.base.pojo.dto.TemplateDetailDTO;
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.io.File;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+/**
+ * @author mozuwen
+ * @date 2024-09-23
+ * @description 净值模板解析
+ */
 @Service
 public class EmailTemplateService {
 
+    private static final Logger log = LoggerFactory.getLogger(EmailTemplateService.class);
+
     private final EmailTemplateMappingMapper emailTemplateMappingMapper;
     private final EmailTemplateApplicationRuleMapper emailTemplateApplicationRuleMapper;
     private final EmailTemplateDataRuleMapper emailTemplateDataRuleMapper;
@@ -51,19 +72,229 @@ public class EmailTemplateService {
         this.emailTemplateDataRuleMapper = emailTemplateDataRuleMapper;
     }
 
-    public List<TemplateDetailDTO> getTemplateDetail(EmailContentInfoDTO emailContentInfoDTO, String filePath) {
+
+    public List<EmailFundNavDTO> parseUsingTemplate(EmailContentInfoDTO emailContentInfoDTO) {
+        // 考虑文件为PDF，html,zip等情况 -> 将正文html和pdf文件转成Excel
+        List<EmailFileContentDTO> emailFileContentDTOList = getRealFilePath(emailContentInfoDTO.getFilePath(), emailContentInfoDTO.getEmailContent());
+        if (CollUtil.isEmpty(emailFileContentDTOList)) {
+            return CollUtil.newArrayList();
+        }
+        List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
+        for (EmailFileContentDTO emailFileContentDTO : emailFileContentDTOList) {
+            String filePath = emailFileContentDTO.getFilePath();
+            String textContent = emailFileContentDTO.getContent();
+            List<TemplateDetailDTO> templateDetailDTOList = getTemplateDetail(emailContentInfoDTO, textContent, filePath);
+            if (CollUtil.isEmpty(templateDetailDTOList)) {
+                return CollUtil.newArrayList();
+            }
+            // 按照模板分别进行解析
+            for (TemplateDetailDTO templateDetailDTO : templateDetailDTOList) {
+                try {
+                    // 按照模板配置规则解析
+                    List<EmailFundNavDTO> fundNavDTOList = extraFundNav(filePath, textContent, templateDetailDTO);
+                    fundNavDTOList = fundNavDTOList.stream().filter(NavDataUtil::navDataFormatCheck).toList();
+                    if (CollUtil.isNotEmpty(fundNavDTOList)) {
+                        emailFundNavDTOList.addAll(fundNavDTOList);
+                    }
+                } catch (Exception e) {
+                    log.error("净值模板解析报错 -> 模板id:{},文件:{},堆栈信息:{}", templateDetailDTO.getTemplateId(), filePath, ExceptionUtil.stacktraceToString(e));
+                }
+            }
+        }
+        // 过滤掉相同的数据
+        return emailFundNavDTOList.stream().distinct().toList();
+    }
+
+    private List<EmailFundNavDTO> extraFundNav(String filePath, String textContent, TemplateDetailDTO templateDetailDTO) {
+        List<TemplateDataRuleDTO> dataRuleDetailList = templateDetailDTO.getDataRuleDetailList();
+        Map<String, String> fieldValueMap = MapUtil.newHashMap(8);
+        List<Map<String, String>> fieldValueMapList = CollUtil.newArrayList();
+        boolean hasParsedTableData = false;
+        Integer direction = templateDetailDTO.getDirection();
+
+        for (TemplateDataRuleDTO templateDataRuleDTO : dataRuleDetailList) {
+            Integer fieldName = templateDataRuleDTO.getFieldName();
+            Integer position = templateDataRuleDTO.getPosition();
+            if (position == 1) {
+                String valueByPattern = getValueByPattern(textContent, templateDataRuleDTO.getFieldRule());
+                fieldValueMap.put(FILE_TYPE_MAP.get(fieldName), valueByPattern);
+            }
+            if (position == 2 && !hasParsedTableData) {
+                hasParsedTableData = true;
+                List<TemplateDataRuleDTO> dataRuleDTOS = dataRuleDetailList.stream().filter(e -> e.getPosition() == 2).toList();
+                fieldValueMapList = parsedTableData(filePath, direction, dataRuleDTOS);
+            }
+        }
+        return buildEmailFundNavDTO(fieldValueMap, fieldValueMapList);
+    }
+
+    private List<Map<String, String>> parsedTableData(String filePath, Integer direction, List<TemplateDataRuleDTO> dataRuleDTOS) {
+        if (StrUtil.isBlank(filePath) || CollUtil.isEmpty(dataRuleDTOS)) {
+            return CollUtil.newArrayList();
+        }
+        Sheet sheet = ExcelUtil.getFirstSheet(filePath);
+        if (sheet == null) {
+            return CollUtil.newArrayList();
+        }
+
+        List<Map<String, String>> fieldValueMapList = CollUtil.newArrayList();
+        Map<String, Pair<Integer, Integer>> fieldPositionMap = MapUtil.newHashMap();
+        Map<String, String> fieldPatternMap = MapUtil.newHashMap();
+        for (TemplateDataRuleDTO dataRuleDTO : dataRuleDTOS) {
+            Integer row = dataRuleDTO.getRow() - 1;
+            int column = columnLetterToIndex(dataRuleDTO.getColumn());
+            Pair<Integer, Integer> pair = new Pair<>(row, column);
+            fieldPositionMap.put(FILE_TYPE_MAP.get(dataRuleDTO.getFieldName()), pair);
+            if (StrUtil.isNotBlank(dataRuleDTO.getFieldRule())) {
+                fieldPatternMap.put(FILE_TYPE_MAP.get(dataRuleDTO.getFieldName()), dataRuleDTO.getFieldRule());
+            }
+        }
+        if (direction.equals(EmailDataDirectionConst.ROW_DIRECTION_TYPE)) {
+            int startRow = fieldPositionMap.values().stream().map(Pair::getKey).min(Integer::compareTo).orElse(0);
+            for (int i = startRow + 1; i <= sheet.getLastRowNum(); i++) {
+                Map<String, String> fieldValueMap = MapUtil.newHashMap(8);
+                Row row = sheet.getRow(i);
+                if (row == null) {
+                    continue;
+                }
+                for (Map.Entry<String, Pair<Integer, Integer>> fieldEntry : fieldPositionMap.entrySet()) {
+                    String fieldName = fieldEntry.getKey();
+                    String fieldRule = fieldPatternMap.get(fieldName);
+                    int columnIndex = fieldEntry.getValue().getValue();
+                    Cell cell = row.getCell(columnIndex);
+                    String cellValue = getValueByPattern(ExcelUtil.getCellValue(cell), fieldRule);
+                    fieldValueMap.put(fieldName, cellValue);
+                }
+                fieldValueMapList.add(fieldValueMap);
+            }
+        }
+        if (direction.equals(EmailDataDirectionConst.COLUMN_DIRECTION_TYPE)) {
+            int startColumn = fieldPositionMap.values().stream().map(Pair::getValue).min(Integer::compareTo).orElse(0);
+            for (int columnNum = startColumn + 1; columnNum < EmailDataDirectionConst.MAX_ROW_COLUMN; columnNum++) {
+                Map<String, String> fieldValueMap = MapUtil.newHashMap();
+                for (Map.Entry<String, Pair<Integer, Integer>> fieldEntry : fieldPositionMap.entrySet()) {
+                    String fieldName = fieldEntry.getKey();
+                    String fieldRule = fieldPatternMap.get(fieldName);
+                    Integer rowIndex = fieldEntry.getValue().getKey();
+                    Integer columnIndex = fieldEntry.getValue().getValue();
+                    Row row = sheet.getRow(rowIndex);
+                    if (row == null) {
+                        continue;
+                    }
+                    Cell cell = row.getCell(columnIndex);
+                    String cellValue = getValueByPattern(ExcelUtil.getCellValue(cell), fieldRule);
+                    fieldValueMap.put(fieldName, cellValue);
+                }
+                fieldValueMapList.add(fieldValueMap);
+            }
+        }
+        // 表头和数据在同一个单元格
+        if (direction.equals(EmailDataDirectionConst.OTHER_DIRECTION_TYPE)) {
+            Map<String, String> fieldValueMap = MapUtil.newHashMap();
+            for (Map.Entry<String, Pair<Integer, Integer>> fieldEntry : fieldPositionMap.entrySet()) {
+                String fieldName = fieldEntry.getKey();
+                String fieldRule = fieldPatternMap.get(fieldName);
+                Integer rowIndex = fieldEntry.getValue().getKey();
+                Integer columnIndex = fieldEntry.getValue().getValue();
+                Row row = sheet.getRow(rowIndex);
+                if (row == null) {
+                    continue;
+                }
+                Cell cell = row.getCell(columnIndex);
+                String cellValue = getValueByPattern(ExcelUtil.getCellValue(cell), fieldRule);
+                fieldValueMap.put(fieldName, cellValue);
+            }
+            fieldValueMapList.add(fieldValueMap);
+        }
+
+        return fieldValueMapList;
+    }
+
+    /**
+     * 根据正则表达式提取内容
+     *
+     * @param text      文本
+     * @param fieldRule 正则表达式
+     * @return 内容
+     */
+    private String getValueByPattern(String text, String fieldRule) {
+        if (StrUtil.isBlank(text) || StrUtil.isBlank(fieldRule)) {
+            return text;
+        }
+        Pattern pattern = Pattern.compile(fieldRule);
+        Matcher matcher = pattern.matcher(text);
+        while (matcher.find()) {
+            return matcher.group(1);
+        }
+        return null;
+    }
+
+    /**
+     * 封装解析的数据为净值数据
+     *
+     * @param textFieldValueMap      从正文表格中解析到的数据
+     * @param excelFieldValueMapList 从正文文本中解析到的数据
+     * @return 净值数据
+     */
+    private List<EmailFundNavDTO> buildEmailFundNavDTO(Map<String, String> textFieldValueMap, List<Map<String, String>> excelFieldValueMapList) {
+        if (MapUtil.isEmpty(textFieldValueMap) && CollUtil.isEmpty(excelFieldValueMapList)) {
+            return CollUtil.newArrayList();
+        }
+        List<EmailFundNavDTO> fundNavDTOList = CollUtil.newArrayList();
+        if (CollUtil.isNotEmpty(excelFieldValueMapList)) {
+            for (Map<String, String> excelFieldValueMap : excelFieldValueMapList) {
+                fundNavDTOList.add(buildEmailFundNavDTO(excelFieldValueMap, textFieldValueMap));
+            }
+        }
+        if (MapUtil.isNotEmpty(textFieldValueMap) && CollUtil.isEmpty(excelFieldValueMapList)) {
+            fundNavDTOList.add(buildEmailFundNavDTO(null, textFieldValueMap));
+        }
+        return fundNavDTOList;
+    }
+
+    private EmailFundNavDTO buildEmailFundNavDTO(Map<String, String> excelFieldValueMap, Map<String, String> textFieldValueMap) {
+        EmailFundNavDTO fundNavDTO = new EmailFundNavDTO();
+        String registerNumber = MapUtil.isNotEmpty(excelFieldValueMap) && StrUtil.isNotBlank(excelFieldValueMap.get(EmailFieldConst.REGISTER_NUMBER))
+                ? excelFieldValueMap.get(EmailFieldConst.REGISTER_NUMBER) : MapUtil.isNotEmpty(textFieldValueMap) ? textFieldValueMap.get(EmailFieldConst.REGISTER_NUMBER) : null;
+        String fundName = MapUtil.isNotEmpty(excelFieldValueMap) && StrUtil.isNotBlank(excelFieldValueMap.get(EmailFieldConst.FUND_NAME))
+                ? excelFieldValueMap.get(EmailFieldConst.FUND_NAME) : MapUtil.isNotEmpty(textFieldValueMap) ? textFieldValueMap.get(EmailFieldConst.FUND_NAME) : null;
+        String priceDate = MapUtil.isNotEmpty(excelFieldValueMap) && StrUtil.isNotBlank(excelFieldValueMap.get(EmailFieldConst.PRICE_DATE))
+                ? excelFieldValueMap.get(EmailFieldConst.PRICE_DATE) : MapUtil.isNotEmpty(textFieldValueMap) ? textFieldValueMap.get(EmailFieldConst.PRICE_DATE) : null;
+        String nav = MapUtil.isNotEmpty(excelFieldValueMap) && StrUtil.isNotBlank(excelFieldValueMap.get(EmailFieldConst.NAV))
+                ? excelFieldValueMap.get(EmailFieldConst.NAV) : MapUtil.isNotEmpty(textFieldValueMap) ? textFieldValueMap.get(EmailFieldConst.NAV) : null;
+        String cumulativeNavWithdrawal = MapUtil.isNotEmpty(excelFieldValueMap) && StrUtil.isNotBlank(excelFieldValueMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL))
+                ? excelFieldValueMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL) : MapUtil.isNotEmpty(textFieldValueMap) ? textFieldValueMap.get(EmailFieldConst.CUMULATIVE_NAV_WITHDRAWAL) : null;
+        String assetShare = MapUtil.isNotEmpty(excelFieldValueMap) && StrUtil.isNotBlank(excelFieldValueMap.get(EmailFieldConst.ASSET_SHARE))
+                ? excelFieldValueMap.get(EmailFieldConst.ASSET_SHARE) : MapUtil.isNotEmpty(textFieldValueMap) ? textFieldValueMap.get(EmailFieldConst.ASSET_SHARE) : null;
+        String assetNet = MapUtil.isNotEmpty(excelFieldValueMap) && StrUtil.isNotBlank(excelFieldValueMap.get(EmailFieldConst.ASSET_NET))
+                ? excelFieldValueMap.get(EmailFieldConst.ASSET_NET) : MapUtil.isNotEmpty(textFieldValueMap) ? textFieldValueMap.get(EmailFieldConst.ASSET_NET) : null;
+
+        fundNavDTO.setRegisterNumber(registerNumber);
+        fundNavDTO.setFundName(fundName);
+        fundNavDTO.setPriceDate(priceDate);
+        fundNavDTO.setNav(nav);
+        fundNavDTO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
+        fundNavDTO.setAssetShare(assetShare);
+        fundNavDTO.setAssetNet(assetNet);
+        return fundNavDTO;
+    }
+
+
+    public List<TemplateDetailDTO> getTemplateDetail(EmailContentInfoDTO emailContentInfoDTO, String textContent, String filePath) {
         List<TemplateDetailDTO> templateDetailDTOList = CollUtil.newArrayList();
         String senderEmail = emailContentInfoDTO.getSenderEmail();
-        if (StrUtil.isBlank(senderEmail) || StrUtil.isBlank(filePath)) {
+        if (StrUtil.isBlank(senderEmail)) {
             return templateDetailDTOList;
         }
+        // 1-excel类型，2-正文类型
+        int type = StrUtil.isNotBlank(emailContentInfoDTO.getFilePath()) && ExcelUtil.isHTML(emailContentInfoDTO.getFilePath()) ? 2 : 1;
         // 查询邮箱配置的模板Id
-        List<EmailTemplateInfoDO> emailTemplateInfoDOList = emailTemplateMappingMapper.queryByEmail(senderEmail);
+        List<EmailTemplateInfoDO> emailTemplateInfoDOList = emailTemplateMappingMapper.queryByEmail(senderEmail, type);
         List<Integer> templateIdList = emailTemplateInfoDOList.stream().map(EmailTemplateInfoDO::getId).collect(Collectors.toList());
         if (CollUtil.isEmpty(templateIdList)) {
             return templateDetailDTOList;
         }
-        Map<Integer, Integer> templateIdDirectionMap = emailTemplateInfoDOList.stream().collect(Collectors.toMap(EmailTemplateInfoDO::getId, EmailTemplateInfoDO::getDirection));
+        Map<Integer, EmailTemplateInfoDO> templateIdDirectionMap = emailTemplateInfoDOList.stream().collect(Collectors.toMap(EmailTemplateInfoDO::getId, v -> v));
         // 查询模版适用性规则 -> 判断邮件是否满足模板适用性规则
         List<EmailTemplateApplicationRuleDO> templateApplicationRuleDOList = emailTemplateApplicationRuleMapper.queryByTemplateId(templateIdList);
         if (CollUtil.isEmpty(templateApplicationRuleDOList)) {
@@ -78,66 +309,111 @@ public class EmailTemplateService {
             Integer templateId = templateIdApplicationRule.getKey();
             List<EmailTemplateApplicationRuleDO> applicationRuleDOList = templateIdApplicationRule.getValue();
             // 判断是否满足模板适用性规则
-            boolean isMatchTemplate = isMatchTemplate(emailContentInfoDTO.getEmailTitle(), emailContentInfoDTO.getFileName(), filePath, applicationRuleDOList);
+            boolean isMatchTemplate = isMatchTemplate(textContent, emailContentInfoDTO.getEmailTitle(), emailContentInfoDTO.getFileName(), filePath, applicationRuleDOList);
             if (!isMatchTemplate) {
                 continue;
             }
             TemplateDetailDTO templateDetailDTO = new TemplateDetailDTO();
             templateDetailDTO.setTemplateId(templateId);
-            // 模板的数据行方向：默认为行
-            Integer directionOrDefault = templateIdDirectionMap.getOrDefault(templateId, 1);
-            templateDetailDTO.setDirection(directionOrDefault);
-            Map<String, Pair<Integer, Integer>> fieldPositionMap = getFieldPosition(templateIdDataRuleMap.get(templateId));
-            templateDetailDTO.setFieldPositionMap(fieldPositionMap);
+            EmailTemplateInfoDO emailTemplateInfoDO = templateIdDirectionMap.get(templateId);
+            templateDetailDTO.setDirection(emailTemplateInfoDO.getDirection());
+            templateDetailDTO.setType(emailTemplateInfoDO.getType());
+            templateDetailDTO.setStartIndex(emailTemplateInfoDO.getStartIndex());
+            templateDetailDTO.setEndIndex(emailTemplateInfoDO.getEndIndex());
+
+            List<EmailTemplateDataRuleDO> dataRuleDOList = templateIdDataRuleMap.get(templateId);
+            List<TemplateDataRuleDTO> dataRuleDetailList = dataRuleDOList.stream().map(e -> BeanUtil.copyProperties(e, TemplateDataRuleDTO.class)).toList();
+            templateDetailDTO.setDataRuleDetailList(dataRuleDetailList);
             templateDetailDTOList.add(templateDetailDTO);
         }
         return templateDetailDTOList;
     }
 
-    private Map<String, Pair<Integer, Integer>> getFieldPosition(List<EmailTemplateDataRuleDO> emailTemplateDataRuleDOList) {
-        Map<String, Pair<Integer, Integer>> fieldPositionMap = new HashMap<>();
-        for (EmailTemplateDataRuleDO templateDataRuleDO : emailTemplateDataRuleDOList) {
-            Integer fieldName = templateDataRuleDO.getFieldName();
-            String fieldValue = FILE_TYPE_MAP.get(fieldName);
-            int column = columnLetterToIndex(templateDataRuleDO.getColumn());
-            Pair<Integer, Integer> pair = new Pair<>(templateDataRuleDO.getRow() - 1, column);
-            fieldPositionMap.put(fieldValue, pair);
+    public List<EmailFileContentDTO> getRealFilePath(String filePath, String content) {
+        List<EmailFileContentDTO> emailFileContentDTOList = CollUtil.newArrayList();
+        if (StrUtil.isBlank(filePath)) {
+            return emailFileContentDTOList;
         }
-        return fieldPositionMap;
-    }
+        if (ExcelUtil.isExcel(filePath)) {
+            emailFileContentDTOList.add(new EmailFileContentDTO(filePath, null));
+        } else if (ExcelUtil.isHTML(filePath)) {
+            String excelFilePath = filePath.replace(".html", ".xlsx");
+            excelFilePath = ExcelUtil.contentConvertToExcel(content, excelFilePath);
+            emailFileContentDTOList.add(new EmailFileContentDTO(excelFilePath, getTextFromHtml(content)));
+        } else if (ExcelUtil.isPdf(filePath)) {
+            String excelFilePath = filePath.replace(".pdf", ".xlsx").replace(".PDF", ".xlsx");
+            excelFilePath = ExcelUtil.pdfConvertToExcel(filePath, excelFilePath);
+            emailFileContentDTOList.add(new EmailFileContentDTO(excelFilePath, null));
+        } else if (ExcelUtil.isZip(filePath)) {
 
-    private boolean isMatchTemplate(String emailTitle, String fileName, String filePath, List<EmailTemplateApplicationRuleDO> applicationRuleDOList) {
-        int notMatchCount = 0;
-        for (EmailTemplateApplicationRuleDO applicationRuleDO : applicationRuleDOList) {
-            if (!isMathApplicationRule(applicationRuleDO, emailTitle, fileName, filePath)) {
-                notMatchCount++;
+            String destPath = filePath.replaceAll(".zip", "").replaceAll(".ZIP", "");
+            List<String> dir = ExcelUtil.extractCompressedFiles(filePath, destPath);
+            for (String zipFilePath : dir) {
+                File file = new File(zipFilePath);
+                if (file.isDirectory()) {
+                    for (String navFilePath : Objects.requireNonNull(file.list())) {
+                        Optional.ofNullable(getRealFilePath(navFilePath, null)).ifPresent(emailFileContentDTOList::addAll);
+                    }
+                } else {
+                    Optional.ofNullable(getRealFilePath(zipFilePath, null)).ifPresent(emailFileContentDTOList::addAll);
+                }
             }
         }
-        return notMatchCount == 0;
+        return emailFileContentDTOList;
+    }
+
+    private String getTextFromHtml(String htmlContent) {
+        if (StrUtil.isBlank(htmlContent)) {
+            return null;
+        }
+        Document doc = Jsoup.parse(htmlContent);
+        if (doc == null) {
+            return htmlContent;
+        }
+        Elements allElements = doc.getAllElements();
+        String textContent = allElements.stream().map(Element::ownText).collect(Collectors.joining("\n"));
+
+        return textContent.trim();
+    }
+
+    /**
+     * 判断邮件是否满足模板的适用性规则
+     *
+     * @param emailContent          邮件正文内容（已转成纯文本,以换行符分割每行内容）
+     * @param emailTitle            邮件主题
+     * @param fileName              附件名称
+     * @param filePath              文件路径（邮件正文html转成）
+     * @param applicationRuleDOList 模板配置的适用性规则列表
+     * @return true or false
+     */
+    private boolean isMatchTemplate(String emailContent, String emailTitle, String fileName,
+                                    String filePath, List<EmailTemplateApplicationRuleDO> applicationRuleDOList) {
+        return applicationRuleDOList.stream()
+                .allMatch(e -> isMathApplicationRule(e, emailContent, emailTitle, fileName, filePath));
     }
 
-    private boolean isMathApplicationRule(EmailTemplateApplicationRuleDO applicationRuleDO, String emailTitle, String fileName, String filePath) {
+    /**
+     * 判断邮件是否满足模板的某条适用性规则
+     *
+     * @param applicationRuleDO 适用性规则
+     * @param emailContent      邮件正文内容（可为空）
+     * @param emailTitle        邮件主题
+     * @param fileName          附件名称
+     * @param filePath          文件路径（邮件正文html转成）
+     * @return true or false
+     */
+    private boolean isMathApplicationRule(EmailTemplateApplicationRuleDO applicationRuleDO, String emailContent, String emailTitle, String fileName, String filePath) {
         boolean isMatch = true;
         Integer type = applicationRuleDO.getType();
         String containKeyword = applicationRuleDO.getContainKeyword();
         String notContainKeyword = applicationRuleDO.getNotContainKeyword();
         if (ApplicationRuleFileConst.EMAIL_TITLE_FILE.equals(type)) {
-            if (StrUtil.isNotBlank(containKeyword)) {
-                isMatch = Arrays.stream(containKeyword.split(",")).allMatch(emailTitle::contains);
-            }
-            if (StrUtil.isNotBlank(notContainKeyword)) {
-                isMatch = Arrays.stream(notContainKeyword.split(",")).noneMatch(emailTitle::contains) && isMatch;
-            }
-        }
-        if (ApplicationRuleFileConst.EMAIL_FILE_NAME_FILE.equals(type)) {
-            if (StrUtil.isNotBlank(containKeyword)) {
-                isMatch = Arrays.stream(containKeyword.split(",")).allMatch(fileName::contains);
-            }
-            if (StrUtil.isNotBlank(notContainKeyword)) {
-                isMatch = Arrays.stream(notContainKeyword.split(",")).noneMatch(fileName::contains) && isMatch;
-            }
-        }
-        if (ApplicationRuleFileConst.EMAIL_EXCEL_CONTENT_FILE.equals(type)) {
+            isMatch = iskeywordMatch(emailTitle, containKeyword, notContainKeyword);
+        } else if (ApplicationRuleFileConst.EMAIL_CONTENT_FILE.equals(type)) {
+            isMatch = iskeywordMatch(emailContent, containKeyword, notContainKeyword);
+        } else if (ApplicationRuleFileConst.EMAIL_FILE_NAME_FILE.equals(type)) {
+            isMatch = iskeywordMatch(fileName, containKeyword, notContainKeyword);
+        } else if (ApplicationRuleFileConst.EMAIL_EXCEL_CONTENT_FILE.equals(type)) {
             if (StrUtil.isNotBlank(fileName) && ExcelUtil.isExcel(fileName)) {
                 if (applicationRuleDO.getRow() == null || StrUtil.isBlank(applicationRuleDO.getColumn())) {
                     return false;
@@ -159,17 +435,30 @@ public class EmailTemplateService {
                 if (StrUtil.isBlank(cellValue)) {
                     return false;
                 }
-                if (StrUtil.isNotBlank(containKeyword)) {
-                    isMatch = Arrays.stream(containKeyword.split(",")).allMatch(cellValue::contains);
-                }
-                if (StrUtil.isNotBlank(notContainKeyword)) {
-                    isMatch = Arrays.stream(notContainKeyword.split(",")).noneMatch(cellValue::contains) && isMatch;
-                }
+                isMatch = iskeywordMatch(cellValue, containKeyword, notContainKeyword);
             }
         }
         return isMatch;
     }
 
+    private Map<String, Pair<Integer, Integer>> getFieldPosition(List<EmailTemplateDataRuleDO> emailTemplateDataRuleDOList) {
+        Map<String, Pair<Integer, Integer>> fieldPositionMap = new HashMap<>();
+        for (EmailTemplateDataRuleDO templateDataRuleDO : emailTemplateDataRuleDOList) {
+            Integer fieldName = templateDataRuleDO.getFieldName();
+            String fieldValue = FILE_TYPE_MAP.get(fieldName);
+            int column = columnLetterToIndex(templateDataRuleDO.getColumn());
+            Pair<Integer, Integer> pair = new Pair<>(templateDataRuleDO.getRow() - 1, column);
+            fieldPositionMap.put(fieldValue, pair);
+        }
+        return fieldPositionMap;
+    }
+
+    /**
+     * 将数据列转成数字型索引
+     *
+     * @param columnName 数据列名：如A,B,....
+     * @return 数据列数字型索引
+     */
     public static int columnLetterToIndex(String columnName) {
         int columnIndex = 0;
         int columnLength = columnName.length();
@@ -184,4 +473,25 @@ public class EmailTemplateService {
         return columnIndex;
     }
 
+    /**
+     * 判断字符串是否包含指定的关键词列表，并且不包含指定的排除关键词列表。
+     *
+     * @param emailTitle        要检查的字符串
+     * @param containKeyword    逗号分隔的必须包含的关键词列表
+     * @param notContainKeyword 逗号分隔的必须不包含的关键词列表
+     * @return true or false
+     */
+    public boolean iskeywordMatch(String emailTitle, String containKeyword, String notContainKeyword) {
+        boolean isMatch = true;
+        if (StrUtil.isNotBlank(containKeyword)) {
+            isMatch &= Arrays.stream(containKeyword.split(","))
+                    .allMatch(emailTitle::contains);
+        }
+        if (StrUtil.isNotBlank(notContainKeyword)) {
+            isMatch &= Arrays.stream(notContainKeyword.split(","))
+                    .noneMatch(emailTitle::contains);
+        }
+        return isMatch;
+    }
+
 }

service-daq/src/main/java/com/simuwang/daq/service/NavEmailParser.java

@@ -2,11 +2,11 @@ package com.simuwang.daq.service;
 
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateUtil;
-import cn.hutool.core.exceptions.ExceptionUtil;
 import cn.hutool.core.lang.Pair;
 import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.util.StrUtil;
 import com.simuwang.base.common.conts.DateConst;
+import com.simuwang.base.common.conts.EmailDataDirectionConst;
 import com.simuwang.base.common.conts.EmailFieldConst;
 import com.simuwang.base.common.conts.EmailTypeConst;
 import com.simuwang.base.common.util.ExcelUtil;
@@ -14,30 +14,15 @@ import com.simuwang.base.common.util.StringUtil;
 import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
 import com.simuwang.base.pojo.dto.EmailFundNavDTO;
 import com.simuwang.base.pojo.dto.FieldPositionDTO;
-import com.simuwang.base.pojo.dto.TemplateDetailDTO;
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
-import org.apache.poi.ss.usermodel.Workbook;
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
-import technology.tabula.*;
-import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
 
 import java.io.File;
-import java.io.FileOutputStream;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -54,14 +39,9 @@ public class NavEmailParser extends AbstractEmailParser {
     @Value("${email.file.path}")
     private String path;
 
-    @Value("${email.force-template.enable}")
-    private boolean forceTemplateEnable;
-
     private final ValuationEmailParser valuationEmailParser;
     private final EmailTemplateService emailTemplateService;
 
-    private static final Integer ROW_DIRECTION_TYPE = 1;
-    private static final Integer COLUMN_DIRECTION_TYPE = 2;
     private static final int MAX_COLUMN = 20;
 
     public NavEmailParser(ValuationEmailParser valuationEmailParser, EmailTemplateService emailTemplateService) {
@@ -79,8 +59,9 @@ public class NavEmailParser extends AbstractEmailParser {
         List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
         String emailContent = emailContentInfoDTO.getEmailContent();
         // 1.解析邮件正文
-        if (StrUtil.isNotBlank(emailContent)) {
-            emailFundNavDTOList = parseEmailContent(emailContentInfoDTO, emailContent, emailFieldMap);
+        if (StrUtil.isNotBlank(emailContentInfoDTO.getFilePath()) && ExcelUtil.isHTML(emailContentInfoDTO.getFileName())) {
+            String excelFilePath = emailContentInfoDTO.getFilePath().replace(".html", ".xlsx");
+            emailFundNavDTOList = parseEmailContent(emailContent, excelFilePath, emailFieldMap);
         }
         // 2.解析邮件excel附件
         if (StrUtil.isNotBlank(emailContentInfoDTO.getFilePath()) && ExcelUtil.isExcel(emailContentInfoDTO.getFileName())) {
@@ -110,8 +91,8 @@ public class NavEmailParser extends AbstractEmailParser {
         }
 
         // 通用模版解析 -> 根据配置的模板进行解析
-        if (CollUtil.isEmpty(emailFundNavDTOList) || forceTemplateEnable) {
-            emailFundNavDTOList = parseUsingTemplate(emailContentInfoDTO);
+        if (CollUtil.isEmpty(emailFundNavDTOList)) {
+            emailFundNavDTOList = emailTemplateService.parseUsingTemplate(emailContentInfoDTO);
         }
         return emailFundNavDTOList;
     }
@@ -149,55 +130,14 @@ public class NavEmailParser extends AbstractEmailParser {
         return fundNavDTOList;
     }
 
-
     private List<EmailFundNavDTO> parsePdfFile(String filePath, String excelFilePath, Map<String, List<String>> emailFieldMap) {
-        excelFilePath = pdfConvertToExcel(filePath, excelFilePath);
+        excelFilePath = ExcelUtil.pdfConvertToExcel(filePath, excelFilePath);
         if (StrUtil.isBlank(excelFilePath)) {
             return CollUtil.newArrayList();
         }
         return parseExcelFile(excelFilePath, emailFieldMap);
     }
 
-    private String pdfConvertToExcel(String filePath, String excelFilePath) {
-        File savefile = new File(excelFilePath);
-        if (!savefile.exists()) {
-            if (!savefile.getParentFile().exists()) {
-                savefile.getParentFile().mkdirs();
-                savefile.getParentFile().setExecutable(true);
-            }
-        }
-        try (OutputStream outputStream = Files.newOutputStream(Paths.get(excelFilePath))) {
-            PDDocument document = Loader.loadPDF(new File(filePath));
-            PageIterator extract = new ObjectExtractor(document).extract();
-            Workbook workbook = new XSSFWorkbook();
-            Sheet sheet = workbook.createSheet("Sheet1");
-            while (extract.hasNext()) {
-                Page next = extract.next();
-                List<Table> tableList = new SpreadsheetExtractionAlgorithm().extract(next);
-                for (Table table : tableList) {
-                    List<List<RectangularTextContainer>> rows = table.getRows();
-                    for (int rowNum = 0; rowNum < rows.size(); rowNum++) {
-                        Row sheetRow = sheet.createRow(rowNum);
-                        List<RectangularTextContainer> textContainerList = rows.get(rowNum);
-                        for (int cellNum = 0; cellNum < textContainerList.size(); cellNum++) {
-                            Cell cell = sheetRow.createCell(cellNum);
-                            RectangularTextContainer textContainer = textContainerList.get(cellNum);
-                            if (textContainer != null) {
-                                cell.setCellValue(textContainer.getText());
-                            }
-                        }
-                    }
-                }
-            }
-            // 将Excel工作簿写入输出流
-            workbook.write(outputStream);
-        } catch (Exception e) {
-            log.error("解析邮件pdf附件报错 -> 堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
-            return null;
-        }
-        return excelFilePath;
-    }
-
     /**
      * 解析邮件excel附件
      *
@@ -207,6 +147,10 @@ public class NavEmailParser extends AbstractEmailParser {
      */
     private List<EmailFundNavDTO> parseExcelFile(String filePath, Map<String, List<String>> emailFieldMap) {
         Sheet sheet = ExcelUtil.getFirstSheet(filePath);
+        if (sheet == null) {
+            log.info("获取不到有效的sheet页面,文件路径：{}", filePath);
+            return CollUtil.newArrayList();
+        }
         // 1.找到表头所在位置
         Map<String, Pair<Integer, Integer>> fieldPositionMap = getFieldPosition(sheet, emailFieldMap);
         if (MapUtil.isEmpty(fieldPositionMap)) {
@@ -225,110 +169,18 @@ public class NavEmailParser extends AbstractEmailParser {
     /**
      * 解析邮件正文
      *
-     * @param emailContentInfoDTO 邮件信息
-     * @param emailContent        正文内容
-     * @param emailFieldMap       邮件字段识别规则映射表
+     * @param emailContent  正文内容
+     * @param emailFieldMap 邮件字段识别规则映射表
      * @return 解析到的净值数据
      */
-    private List<EmailFundNavDTO> parseEmailContent(EmailContentInfoDTO emailContentInfoDTO, String emailContent, Map<String, List<String>> emailFieldMap) {
-        String excelFilePath = contentConvertToExcel(emailContentInfoDTO);
+    private List<EmailFundNavDTO> parseEmailContent(String emailContent, String excelFilePath, Map<String, List<String>> emailFieldMap) {
+        excelFilePath = ExcelUtil.contentConvertToExcel(emailContent, excelFilePath);
         if (StrUtil.isBlank(excelFilePath)) {
             return CollUtil.newArrayList();
         }
         return parseExcelFile(excelFilePath, emailFieldMap);
     }
 
-    public String contentConvertToExcel(EmailContentInfoDTO emailContentInfoDTO) {
-        Document doc = Jsoup.parse(emailContentInfoDTO.getEmailContent());
-        Element table = doc.select("table").first();
-        Elements rows = table.select("tr");
-        String excelFilePath = path + "/content/" + emailContentInfoDTO.getEmailAddress() + "/" + emailContentInfoDTO.getEmailDate().substring(0, 10).replaceAll("-", "") + "/"
-                + emailContentInfoDTO.getFileName().replace(".html", ".xlsx");
-        File saveFile = new File(excelFilePath);
-        if (!saveFile.exists()) {
-            if (!saveFile.getParentFile().exists()) {
-                saveFile.getParentFile().mkdirs();
-                saveFile.getParentFile().setExecutable(true);
-            }
-        }
-        try (OutputStream outputStream = new FileOutputStream(saveFile)) {
-            // 创建一个新的Excel工作簿
-            Workbook workbook = new XSSFWorkbook();
-            Sheet sheet = workbook.createSheet("Sheet1");
-            ExcelUtil.writeDataToSheet(sheet, rows);
-            // 将Excel工作簿写入输出流
-            workbook.write(outputStream);
-        } catch (Exception e) {
-            log.error("解析正文报错 -> 邮件主题:{},邮件日期:{},堆栈信息:{}", emailContentInfoDTO.getEmailTitle(), emailContentInfoDTO.getEmailDate(), ExceptionUtil.stacktraceToString(e));
-            return null;
-        }
-        return excelFilePath;
-    }
-
-    private List<EmailFundNavDTO> parseUsingTemplate(EmailContentInfoDTO emailContentInfoDTO) {
-        // 考虑文件为PDF，html,zip等情况 -> 将相关文件转成Excel
-        List<String> filePathList = getRealFilePath(emailContentInfoDTO.getFilePath(), emailContentInfoDTO);
-        if (CollUtil.isEmpty(filePathList)) {
-            return CollUtil.newArrayList();
-        }
-        List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
-        for (String filePath : filePathList) {
-            List<TemplateDetailDTO> templateDetailDTOList = emailTemplateService.getTemplateDetail(emailContentInfoDTO, filePath);
-            if (CollUtil.isEmpty(templateDetailDTOList) || ExcelUtil.getFirstSheet(filePath) == null) {
-                return CollUtil.newArrayList();
-            }
-            // 获取文件的sheet页
-            Sheet sheet = ExcelUtil.getFirstSheet(filePath);
-            // 按照模板分别进行解析 -> 只要有一个模板解析到数据就停止
-            for (TemplateDetailDTO templateDetailDTO : templateDetailDTOList) {
-                try {
-                    Integer direction = templateDetailDTO.getDirection();
-                    Map<String, Pair<Integer, Integer>> fieldPositionMap = templateDetailDTO.getFieldPositionMap();
-                    List<EmailFundNavDTO> fundNavDTOList = parseSheetData(sheet, fieldPositionMap, direction);
-                    fundNavDTOList = fundNavDTOList.stream().filter(super::dataFormat).collect(Collectors.toList());
-                    if (CollUtil.isNotEmpty(fundNavDTOList)) {
-                        emailFundNavDTOList.addAll(fundNavDTOList);
-                        break;
-                    }
-                } catch (Exception e) {
-                    log.error("净值模板解析报错 -> 模板id:{},文件:{},堆栈信息:{}", templateDetailDTO.getTemplateId(), filePath, ExceptionUtil.stacktraceToString(e));
-                }
-            }
-        }
-        return emailFundNavDTOList;
-    }
-
-    private List<String> getRealFilePath(String filePath, EmailContentInfoDTO emailContentInfoDTO) {
-        List<String> filePathList = CollUtil.newArrayList();
-        if (StrUtil.isBlank(filePath)) {
-            return filePathList;
-        }
-        if (ExcelUtil.isExcel(filePath)) {
-            filePathList.add(filePath);
-        } else if (ExcelUtil.isHTML(filePath)) {
-            String excelFilePath = contentConvertToExcel(emailContentInfoDTO);
-            Optional.ofNullable(excelFilePath).ifPresent(filePathList::add);
-        } else if (ExcelUtil.isPdf(filePath)) {
-            String excelFilePath = filePath.replace(".pdf", ".xlsx").replace(".PDF", ".xlsx");
-            excelFilePath = pdfConvertToExcel(filePath, excelFilePath);
-            Optional.ofNullable(excelFilePath).ifPresent(filePathList::add);
-        } else if (ExcelUtil.isZip(filePath)) {
-            String destPath = filePath.replaceAll(".zip", "").replaceAll(".ZIP", "");
-            List<String> dir = ExcelUtil.extractCompressedFiles(filePath, destPath);
-            for (String zipFilePath : dir) {
-                File file = new File(zipFilePath);
-                if (file.isDirectory()) {
-                    for (String navFilePath : Objects.requireNonNull(file.list())) {
-                        Optional.ofNullable(getRealFilePath(navFilePath, emailContentInfoDTO)).ifPresent(filePathList::addAll);
-                    }
-                } else {
-                    Optional.ofNullable(getRealFilePath(zipFilePath, emailContentInfoDTO)).ifPresent(filePathList::addAll);
-                }
-            }
-        }
-        return filePathList;
-    }
-
     /**
      * 根据字段所在表格的位置提取净值数据
      *
@@ -340,12 +192,12 @@ public class NavEmailParser extends AbstractEmailParser {
     private List<EmailFundNavDTO> parseSheetData(Sheet sheet, Map<String, Pair<Integer, Integer>> fieldPositionMap, Integer direction) {
         List<EmailFundNavDTO> fundNavDTOList = CollUtil.newArrayList();
         // 通过表头所在位置判断是行数据还是列数据
-        Integer dataDirectionType = direction != null ? direction : detectDataDirection(fieldPositionMap);
+        Integer dataDirectionType = direction != null ? direction : ExcelUtil.detectDataDirection(fieldPositionMap);
         // 数据起始行,起始列
-        int initRow = dataDirectionType.equals(ROW_DIRECTION_TYPE) ? fieldPositionMap.values().stream().map(Pair::getKey).max(Integer::compareTo).orElse(0)
+        int initRow = dataDirectionType.equals(EmailDataDirectionConst.ROW_DIRECTION_TYPE) ? fieldPositionMap.values().stream().map(Pair::getKey).max(Integer::compareTo).orElse(0)
                 : fieldPositionMap.values().stream().map(Pair::getKey).min(Integer::compareTo).orElse(0);
         int initColumn = fieldPositionMap.values().stream().map(Pair::getValue).min(Integer::compareTo).orElse(0);
-        if (dataDirectionType.equals(ROW_DIRECTION_TYPE)) {
+        if (dataDirectionType.equals(EmailDataDirectionConst.ROW_DIRECTION_TYPE)) {
             // 表头字段-列号映射关系
             Map<String, Integer> fieldColumnMap = getFieldRow(fieldPositionMap);
             int lastRowNum = sheet.getLastRowNum();
@@ -355,12 +207,12 @@ public class NavEmailParser extends AbstractEmailParser {
                 Optional.ofNullable(readSheetRowData(sheetRow, fieldColumnMap)).ifPresent(fundNavDTOList::addAll);
             }
         }
-        if (dataDirectionType.equals(COLUMN_DIRECTION_TYPE)) {
+        if (dataDirectionType.equals(EmailDataDirectionConst.COLUMN_DIRECTION_TYPE)) {
             // 表头字段-行号映射关系
             Map<Integer, String> fieldRowMap = getRowField(fieldPositionMap);
             int lastRow = fieldPositionMap.values().stream().map(Pair::getKey).max(Integer::compareTo).orElse(0);
             // 遍历每一列
-            for (int columnNum = initColumn + 1; columnNum < MAX_COLUMN; columnNum++) {
+            for (int columnNum = initColumn + 1; columnNum < EmailDataDirectionConst.MAX_ROW_COLUMN; columnNum++) {
                 Map<String, String> fieldValueMap = MapUtil.newHashMap();
                 for (int rowNum = initRow; rowNum <= lastRow; rowNum++) {
                     Row row = sheet.getRow(rowNum);
@@ -561,17 +413,6 @@ public class NavEmailParser extends AbstractEmailParser {
     }
 
     /**
-     * 通过表头所在位置判断是行数据还是列数据
-     *
-     * @param fieldPositionMap excel中表头所在的位置
-     * @return 行方向-1,,列方向-2
-     */
-    private Integer detectDataDirection(Map<String, Pair<Integer, Integer>> fieldPositionMap) {
-        long count = fieldPositionMap.values().stream().map(Pair::getValue).distinct().count();
-        return count == 1 ? COLUMN_DIRECTION_TYPE : ROW_DIRECTION_TYPE;
-    }
-
-    /**
      * 找出excel中表头所在的位置
      *
      * @param sheet         表格工作簿
@@ -636,9 +477,11 @@ public class NavEmailParser extends AbstractEmailParser {
         // 母基金缺少代码的情况
         if (hasParentField && fieldPositionMap.get(EmailFieldConst.PARENT_REGISTER_NUMBER) == null) {
             List<FieldPositionDTO> fieldPositionDTOS = tempFieldPositionMap.get(EmailFieldConst.REGISTER_NUMBER);
-            Pair<Integer, Integer> parentRegisterNumberPair = fieldPositionDTOS.stream()
-                    .filter(e -> e.getFieldValue().contains("协会") || e.getFieldValue().contains("备案")).map(FieldPositionDTO::getPair).findFirst().orElse(null);
-            fieldPositionMap.put(EmailFieldConst.PARENT_REGISTER_NUMBER, parentRegisterNumberPair);
+            if(CollUtil.isNotEmpty(fieldPositionDTOS)) {
+                Pair<Integer, Integer> parentRegisterNumberPair = fieldPositionDTOS.stream()
+                        .filter(e -> e.getFieldValue().contains("协会") || e.getFieldValue().contains("备案")).map(FieldPositionDTO::getPair).findFirst().orElse(null);
+                fieldPositionMap.put(EmailFieldConst.PARENT_REGISTER_NUMBER, parentRegisterNumberPair);
+            }
         }
         return fieldPositionMap;
     }

service-deploy/src/main/resources/application.yml

@@ -70,8 +70,6 @@ email-rule:
 email:
   file:
     path: /data/file/nav
-  force-template:
-    enable: true
 
 # 配置
 simuwang:

service-deploy/src/test/java/com/simuwang/ApplicationTest.java

@@ -37,8 +37,8 @@ public class ApplicationTest {
 //        emailInfoDTO.setPort("993");
 //        emailInfoDTO.setProtocol("imap");
 
-        Date startDate = DateUtil.parse("2024-09-20 09:02:00", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-09-20 10:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-09-24 11:23:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2024-09-24 11:25:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {
@@ -48,7 +48,7 @@ public class ApplicationTest {
 
     @Test
     public void testReparseEmail() {
-        emailParseApiService.reparseEmail(7);
+        emailParseApiService.reparseEmail(593);
     }
 
     @Test