feat:邮件解析-调整文件存储路径及更改基金信息表表名

service-base/src/main/java/com/simuwang/base/common/util/ExcelUtil.java

@@ -207,7 +207,7 @@ public class ExcelUtil {
                 } else {
                     try (FileOutputStream fos = new FileOutputStream(entryFile)) {
                         IOUtils.copy(ais, fos);
-                        filePathList.add(entryFile.getAbsolutePath());
+                        filePathList.add(entryFile.getPath());
                     }
                 }
             }

service-base/src/main/java/com/simuwang/base/mapper/FundInfomationMapper.java

@@ -14,7 +14,7 @@ import java.util.Map;
  * Description: ${DESCRIPTION}
  */
 @Mapper
-public interface FundInfomationMapper {
+public interface FundInfoMapper {
     List<Map<String, String>> searchFundInfoByKeyword(@Param("keyword") String keyword);
 
     List<FundInformationVO> searchFundInfoList(@Param("fundId") String fundId, @Param("fundName") String fundName,

service-base/src/main/java/com/simuwang/base/pojo/dos/FundInformationDO.java

@@ -14,8 +14,8 @@ import java.util.Date;
  * Description: ${DESCRIPTION}
  */
 @Data
-@TableName("fund_information")
-public class FundInformationDO {
+@TableName("pvn_fund_info")
+public class FundInfoDO {
     /**
      * 主键Id
      */

service-base/src/main/resources/mapper/FundInformationMapper.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="com.simuwang.base.mapper.FundInfomationMapper">
-    <resultMap id="BaseResultMap" type="com.simuwang.base.pojo.dos.FundInformationDO">
+<mapper namespace="com.simuwang.base.mapper.FundInfoMapper">
+    <resultMap id="BaseResultMap" type="com.simuwang.base.pojo.dos.FundInfoDO">
         <id column="id" property="id"/>
         <result column="p_fund_id" property="pFundId"/>
         <result column="fund_name" property="fundName"/>
@@ -31,7 +31,7 @@
         select f.fund_id as fundId,
         f.fund_short_name as fundShortName,
         f.register_number as registerNumber
-        from PPW_EMAIL.fund_information f where f.isvalid =1
+        from PPW_EMAIL.pvn_fund_info f where f.isvalid =1
         <if test="keyword != null and keyword !=''">
             and (f.fund_name like concat('%',#{keyword},'%') or f.fund_short_name like concat('%',#{keyword},'%') or f.register_number like concat('%',#{keyword},'%'))
         </if>
@@ -45,7 +45,7 @@
                r.nav_frequency as navFrequency,
                r.asset_Frequency as assetFrequency,
                r.frequency_remark as frequencyRemark
-        from PPW_EMAIL.fund_information f
+        from PPW_EMAIL.pvn_fund_info f
         join PPW_EMAIL.company_information c on f.trust_id = c.company_id
         left join (select nv.fund_id,max(nv.price_date) as last_price_date from PPW_EMAIL.nav nv where nv.isvalid =1 group by nv.fund_id) n
         on f.fund_id = n .fund_id
@@ -76,7 +76,7 @@
 
     <select id="queryFundByNameAndRegisterNumber" resultType="java.lang.String">
         select fund_id
-        from PPW_EMAIL.fund_information
+        from PPW_EMAIL.pvn_fund_info
         where isvalid = 1
           and fund_name = #{fundName}
           and register_number = #{registerNumber}
@@ -84,14 +84,14 @@
 
     <select id="queryFundByName" resultType="java.lang.String">
         select fund_id
-        from PPW_EMAIL.fund_information
+        from PPW_EMAIL.pvn_fund_info
         where isvalid = 1
           and fund_name = #{fundName}
     </select>
 
     <select id="queryFundByRegisterNumber" resultType="java.lang.String">
         select fund_id
-        from PPW_EMAIL.fund_information
+        from PPW_EMAIL.pvn_fund_info
         where isvalid = 1
           and register_number = #{registerNumber}
     </select>

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -26,6 +26,7 @@ import jakarta.mail.search.ReceivedDateTerm;
 import jakarta.mail.search.SearchTerm;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.File;
@@ -43,12 +44,15 @@ public class EmailParseService {
 
     private static final Logger log = LoggerFactory.getLogger(EmailParseService.class);
 
+    @Value("${email.file.path}")
+    private String path;
+
     private final EmailTypeRuleMapper emailTypeRuleMapper;
     private final EmailRuleConfig emailRuleConfig;
     private final EmailFieldMappingMapper emailFieldMapper;
     private final EmailParserFactory emailParserFactory;
     private final EmailParseInfoMapper emailParseInfoMapper;
-    private final FundInfomationMapper fundInfomationMapper;
+    private final FundInfoMapper fundInfoMapper;
     private final EmailFileInfoMapper emailFileInfoMapper;
     private final EmailFundNavMapper emailFundNavMapper;
     private final EmailFundAssetMapper emailFundAssetMapper;
@@ -56,7 +60,7 @@ public class EmailParseService {
 
     public EmailParseService(EmailTypeRuleMapper emailTypeRuleMapper, EmailRuleConfig emailRuleConfig,
                              EmailFieldMappingMapper emailFieldMapper, EmailParserFactory emailParserFactory,
-                             EmailParseInfoMapper emailParseInfoMapper, FundInfomationMapper fundInfomationMapper,
+                             EmailParseInfoMapper emailParseInfoMapper, FundInfoMapper fundInfoMapper,
                              EmailFileInfoMapper emailFileInfoMapper, EmailFundNavMapper emailFundNavMapper,
                              EmailFundAssetMapper emailFundAssetMapper, FundAliasMapper fundAliasMapper) {
         this.emailTypeRuleMapper = emailTypeRuleMapper;
@@ -64,7 +68,7 @@ public class EmailParseService {
         this.emailFieldMapper = emailFieldMapper;
         this.emailParserFactory = emailParserFactory;
         this.emailParseInfoMapper = emailParseInfoMapper;
-        this.fundInfomationMapper = fundInfomationMapper;
+        this.fundInfoMapper = fundInfoMapper;
         this.emailFileInfoMapper = emailFileInfoMapper;
         this.emailFundNavMapper = emailFundNavMapper;
         this.emailFundAssetMapper = emailFundAssetMapper;
@@ -296,7 +300,7 @@ public class EmailParseService {
 
     private List<String> matchFund(String fundName, String registerNumber) {
         // 1.基金名称 + 备案编码 一起进行匹配
-        List<String> fundIdList = fundInfomationMapper.queryFundByNameAndRegisterNumber(fundName, registerNumber);
+        List<String> fundIdList = fundInfoMapper.queryFundByNameAndRegisterNumber(fundName, registerNumber);
         if (CollUtil.isNotEmpty(fundIdList)) {
             return fundIdList;
         }
@@ -306,7 +310,7 @@ public class EmailParseService {
         }
         // 2.基金名称匹配
         if (StrUtil.isNotBlank(fundName)) {
-            List<String> fundIds = fundInfomationMapper.queryFundByName(fundName);
+            List<String> fundIds = fundInfoMapper.queryFundByName(fundName);
             if (CollUtil.isNotEmpty(fundIds)) {
                 return fundIdList;
             }
@@ -317,7 +321,7 @@ public class EmailParseService {
         }
         // 3.备案编码匹配
         if (StrUtil.isNotBlank(fundName)) {
-            List<String> fundIds = fundInfomationMapper.queryFundByRegisterNumber(registerNumber);
+            List<String> fundIds = fundInfoMapper.queryFundByRegisterNumber(registerNumber);
             if (CollUtil.isNotEmpty(fundIds)) {
                 return fundIds;
             }
@@ -408,7 +412,6 @@ public class EmailParseService {
         // 获取邮件日期大于等于startDate的邮件(搜索条件只支持按天)
         SearchTerm startDateTerm = new ReceivedDateTerm(ComparisonTerm.GE, startDate);
         Message[] messages = folder.search(startDateTerm);
-        String path = "/data/file/nav";
         Map<String, List<EmailContentInfoDTO>> emailMessageMap = MapUtil.newHashMap();
         for (Message message : messages) {
             List<EmailContentInfoDTO> emailContentInfoDTOList = CollUtil.newArrayList();
@@ -439,8 +442,8 @@ public class EmailParseService {
                     emailContentInfoDTO.setEmailContent(content.toString());
                     emailContentInfoDTO.setEmailDate(emailDateStr);
                     String fileName = message.getSubject() + DateUtil.format(emailDate, DateConst.YYYYMMDDHHMMSS24);
-                    String filePath = path + mailboxInfoDTO.getAccount() + "/" + fileName + ".html";
-                    File savefile = new File(path + filePath);
+                    String filePath = path + mailboxInfoDTO.getAccount() + "/" + DateUtil.format(emailDate, DateConst.YYYY_MM_DD) + "/" + fileName + ".html";
+                    File savefile = new File(filePath);
                     savefile.setReadable(true);
                     if (!savefile.exists()) {
                         if (!savefile.getParentFile().exists()) {

service-daq/src/main/java/com/simuwang/daq/service/NavEmailParser.java

@@ -26,6 +26,7 @@ import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import technology.tabula.*;
 import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
@@ -48,6 +49,9 @@ public class NavEmailParser extends AbstractEmailParser {
 
     private static final Logger log = LoggerFactory.getLogger(AbstractEmailParser.class);
 
+    @Value("${email.file.path}")
+    private String path;
+
     private static final Integer ROW_DIRECTION_TYPE = 1;
     private static final Integer COLUMN_DIRECTION_TYPE = 2;
 
@@ -73,14 +77,14 @@ public class NavEmailParser extends AbstractEmailParser {
         }
         // 3.解析邮件pdf附件
         if (StrUtil.isNotBlank(emailContentInfoDTO.getFilePath()) && ExcelUtil.isPdf(emailContentInfoDTO.getFileName())) {
-            String excelFilePath = "/data/file/pdf/" + emailContentInfoDTO.getEmailAddress() + "/" + emailContentInfoDTO.getEmailDate().substring(0, 10).replaceAll("-", "")
+            String excelFilePath = path + emailContentInfoDTO.getEmailAddress() + "/" + emailContentInfoDTO.getEmailDate().substring(0, 10).replaceAll("-", "")
                     + "/" + emailContentInfoDTO.getFileName().replace(".pdf", ".xlsx").replace(".PDF", ".xlsx");
             List<EmailFundNavDTO> fundNavDTOList = parsePdfFile(emailContentInfoDTO.getFilePath(), excelFilePath, emailFieldMap);
             Optional.ofNullable(fundNavDTOList).ifPresent(emailFundNavDTOList::addAll);
         }
         // 4.解析邮件zip,rar附件
         if (StrUtil.isNotBlank(emailContentInfoDTO.getFilePath()) && ExcelUtil.isZip(emailContentInfoDTO.getFileName())) {
-            List<EmailFundNavDTO> fundNavDTOList = parsePackageFile(emailContentInfoDTO, emailContentInfoDTO.getFilePath(), emailFieldMap);
+            List<EmailFundNavDTO> fundNavDTOList = parsePackageFile(emailContentInfoDTO, emailContentInfoDTO.getFileName(), emailContentInfoDTO.getFilePath(), emailFieldMap);
             Optional.ofNullable(fundNavDTOList).ifPresent(emailFundNavDTOList::addAll);
         }
 
@@ -91,8 +95,9 @@ public class NavEmailParser extends AbstractEmailParser {
         return emailFundNavDTOList;
     }
 
-    private List<EmailFundNavDTO> parsePackageFile(EmailContentInfoDTO emailContentInfoDTO, String filePath, Map<String, List<String>> emailFieldMap) {
-        String destPath = filePath.substring(0, filePath.lastIndexOf("/") + 1) + "/" + UUID.randomUUID().toString().replaceAll("-", "");
+    private List<EmailFundNavDTO> parsePackageFile(EmailContentInfoDTO emailContentInfoDTO, String fileName, String filePath, Map<String, List<String>> emailFieldMap) {
+        String destPath = filePath.substring(0, filePath.indexOf(fileName)) + fileName.replaceAll(".zip","").replaceAll(".ZIP","") ;
+        log.info("压缩包地址:{},解压后文件地址:{}", filePath, destPath);
         List<String> dir = ExcelUtil.extractCompressedFiles(filePath, destPath);
         List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
         for (String zipFilePath : dir) {
@@ -117,7 +122,8 @@ public class NavEmailParser extends AbstractEmailParser {
             fundNavDTOList = parseExcelFile(zipFilePath, emailFieldMap);
         }
         if (ExcelUtil.isZip(zipFilePath)) {
-            fundNavDTOList = parsePackageFile(emailContentInfoDTO, zipFilePath, emailFieldMap);
+            String name = new File(zipFilePath).getName();
+            fundNavDTOList = parsePackageFile(emailContentInfoDTO, name, zipFilePath, emailFieldMap);
         }
         return fundNavDTOList;
     }

service-deploy/src/main/resources/application.yml

@@ -44,6 +44,10 @@ email-rule:
   valuation: "估值表"
   report: "月报,年报,季报,报告（待确认）信息披露报告,证券类月报"
 
+email:
+  file:
+    path: /data/file/nav
+
 # 配置
 simuwang:
   token-expire: 86400

service-deploy/src/main/test/java/com/simuwang/datadaq/DataTrusteeApplicationTests.java

@@ -32,7 +32,7 @@ class DataTrusteeApplicationTests {
         emailInfoDTO.setProtocol("imap");
         Map<Integer, List<String>> emailTypeMap = MapUtil.newHashMap();
         emailTypeMap.put(1, List.of("净值"));
-        Date startDate = DateUtil.parse("2024-09-11 11:13:00", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-09-11 11:10:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         Date endDate = DateUtil.parse("2024-09-11 12:00:00", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);

service-manage/src/main/java/com/simuwang/manage/service/impl/FundInformationServiceImpl.java

@@ -1,6 +1,6 @@
 package com.simuwang.manage.service.impl;
 
-import com.simuwang.base.mapper.FundInfomationMapper;
+import com.simuwang.base.mapper.FundInfoMapper;
 import com.simuwang.base.pojo.vo.FundInformationVO;
 import com.simuwang.manage.service.FundInformationService;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -18,15 +18,15 @@ import java.util.Map;
 @Service
 public class FundInformationServiceImpl implements FundInformationService {
     @Autowired
-    private FundInfomationMapper fundInfomationMapper;
+    private FundInfoMapper fundInfoMapper;
     @Override
     public List<Map<String, String>> searchFundInfo(String keyword) {
-        return fundInfomationMapper.searchFundInfoByKeyword(keyword);
+        return fundInfoMapper.searchFundInfoByKeyword(keyword);
     }
 
     @Override
     public List<FundInformationVO> searchFundInfoList(String fundId, String fundName, String companyName, Integer navFrequency, Integer assetFrequency, Integer startDate, Integer endDate) {
-        List<FundInformationVO> result = fundInfomationMapper.searchFundInfoList(fundId,fundName,companyName,navFrequency,assetFrequency,startDate,endDate);
+        List<FundInformationVO> result = fundInfoMapper.searchFundInfoList(fundId,fundName,companyName,navFrequency,assetFrequency,startDate,endDate);
         return result;
     }
 }