feat: 数据看板接口开发

service-base/src/main/java/com/simuwang/base/mapper/AssetMapper.java

@@ -23,4 +23,6 @@ public interface AssetMapper {
     List<AssetDO> selectAssetByFundId(@Param("fundId")String fundId);
 
     void batchDeleteAsset(@Param("fundId")String sourceFundId,@Param("priceDateList") List<String> priceDateList);
+
+    Long countAssetTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/CompanyInformationMapper.java

@@ -17,4 +17,6 @@ public interface CompanyInformationMapper {
     public List<Map<String, String>> searchCompanyInfoByKeyword(@Param("keyword") String keyword);
 
     String getCompanyNameByFundId(@Param("fundId") String fundId);
+
+    Long countCompanyTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/DistributionMapper.java

@@ -37,4 +37,6 @@ public interface DistributionMapper {
     List<DistributionDO> getDistributionByFundId(@Param("fundId")String fundId,@Param("distributeType") DistributeType distributeType);
 
     BigDecimal getSumDistributeByFundId(@Param("fundId")String fundId,@Param("distributeDate") String distributeDate);
+
+    Long countDistributionTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailFundAssetMapper.java

@@ -29,4 +29,6 @@ public interface EmailFundAssetMapper {
     List<EmailParseDetailDO> selectFundAssetByFielId(@Param("fileId") Integer fileId, @Param("priceDate") String priceDate);
 
     List<EmailFundAssetDO> selectAssetByFundId(@Param("fundId") String sourceFundId);
+
+    Long countEmailAssetTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailFundNavMapper.java

@@ -33,4 +33,6 @@ public interface EmailFundNavMapper {
     List<EmailParseDetailDO> selectFundNavByFielId(@Param("fileId") Integer fileId, @Param("priceDate") String priceDate);
 
     List<EmailFundNavDO> selectNavByFundId(@Param("fundId") String sourceFundId);
+
+    Long countEmailNavTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailParseInfoMapper.java

@@ -2,6 +2,7 @@ package com.simuwang.base.mapper;
 
 import com.simuwang.base.pojo.dos.EmailParseInfoDO;
 import com.simuwang.base.pojo.dto.EmailInfoDTO;
+import com.simuwang.base.pojo.dto.query.DataboardQuery;
 import com.simuwang.base.pojo.dto.query.EmailParseQuery;
 import com.simuwang.base.pojo.vo.EmailParseInfoVO;
 import org.apache.ibatis.annotations.Mapper;
@@ -9,6 +10,7 @@ import org.apache.ibatis.annotations.Param;
 
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
 @Mapper
 public interface EmailParseInfoMapper {
@@ -30,4 +32,12 @@ public interface EmailParseInfoMapper {
     void updateParseTime(@Param("id") Integer id, @Param("parseDate") Date parseDate);
 
     List<EmailInfoDTO> queryValuationEmailByFileId(@Param("fileIdList") List<Integer> fileIdList);
+
+    List<Map<String, Object>> searchEmailDataBoard(DataboardQuery databoardQuery);
+
+    List<Map<String, Object>> searchEmailTypeCount(DataboardQuery databoardQuery);
+
+    Long countpdfNoData(@Param("item") DataboardQuery databoardQuery, @Param("errorInfo")String errorInfo);
+
+    Long countEmailTotal(@Param("emailType")Integer emailType);
 }

service-base/src/main/java/com/simuwang/base/mapper/FundInfoMapper.java

@@ -49,4 +49,6 @@ public interface FundInfoMapper {
     String getInceptionDateByFundId(@Param("fundId")String fundId);
 
     FundInfoDO searchFundDetail(@Param("fundId")String fundId);
+
+    Long countFundTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/NavMapper.java

@@ -28,4 +28,6 @@ public interface NavMapper {
     List<NavDO> selectNavByFundId(@Param("fundId")String fundId);
 
     void batchDeleteNav(@Param("fundId")String sourceFundId, @Param("priceDateList")List<String> priceDateList);
+
+    Long countNavTotal();
 }

service-base/src/main/java/com/simuwang/base/pojo/dto/query/DataboardQuery.java

@@ -0,0 +1,58 @@
+package com.simuwang.base.pojo.dto.query;
+
+/**
+ * FileName: DataboardQuery
+ * Author:   chenjianhua
+ * Date:     2024/10/14 10:38
+ * Description: ${DESCRIPTION}
+ */
+public class DataboardQuery {
+    /**
+     * 邮箱地址
+     */
+    private String email;
+    /**
+     * 邮箱类型
+     */
+    private Integer emailType;
+    /**
+     * 开始时间
+     */
+    private String startDate;
+    /**
+     * 结束时间
+     */
+    private String endDate;
+
+    public String getEmail() {
+        return email;
+    }
+
+    public void setEmail(String email) {
+        this.email = email;
+    }
+
+    public Integer getEmailType() {
+        return emailType;
+    }
+
+    public void setEmailType(Integer emailType) {
+        this.emailType = emailType;
+    }
+
+    public String getStartDate() {
+        return startDate;
+    }
+
+    public void setStartDate(String startDate) {
+        this.startDate = startDate;
+    }
+
+    public String getEndDate() {
+        return endDate;
+    }
+
+    public void setEndDate(String endDate) {
+        this.endDate = endDate;
+    }
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/EmailParseCountBoardVO.java

@@ -0,0 +1,20 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: EmailParseCountBoardVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 12:50
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class EmailParseCountBoardVO {
+
+    private Integer total;
+
+    private Integer success;
+
+    private Integer fail;
+
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/EmailParseDataViewVO.java

@@ -0,0 +1,58 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: EmailParseCountBoardVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 12:50
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class EmailParseDataViewVO {
+    /**
+     * 邮件数量
+     */
+    private Long emailNum;
+    /**
+     * 净值邮件数量
+     */
+    private Long navEmailNum;
+    /**
+     * 估值表邮件数量
+     */
+    private Long valuationEmailNum;
+    /**
+     * 定期报告邮件数量
+     */
+    private Long reportEmailNum;
+    /**
+     * 解析净值数量
+     */
+    private Long parseNavNum;
+    /**
+     * 解析规模数量
+     */
+    private Long parseAssetNum;
+    /**
+     * 入库净值数量
+     */
+    private Long NavNum;
+    /**
+     * 入库规模数量
+     */
+    private Long assetNum;
+    /**
+     * 分红数量
+     */
+    private Long distribute;
+    /**
+     * 基金数量
+     */
+    private Long fundNum;
+    /**
+     * 公司数量
+     */
+    private Long companyNum;
+
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/EmailParseFailAnalysisVO.java

@@ -0,0 +1,25 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: EmailParseFailAnalysisVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 13:39
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class EmailParseFailAnalysisVO {
+    /**
+     * 净值类型邮件返回结构
+     */
+    private NavFailAnalysisVO navFailAnalysisVO;
+    /**
+     * 估值表类型邮件返回结构
+     */
+    private ValuationFailAnalysisVO valuationFailAnalysisVO;
+    /**
+     * 报告类型邮件返回结构
+     */
+    private ReportFailAnalysisVO reportFailAnalysisVO;
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/EmailParseTypeBoardVO.java

@@ -0,0 +1,30 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: EmailParseCountBoardVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 12:50
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class EmailParseTypeBoardVO {
+    /**
+     * 总数
+     */
+    private Integer total;
+    /**
+     * 估值表总数
+     */
+    private Integer valuation;
+    /**
+     * 净值总数
+     */
+    private Integer nav;
+    /**
+     * 报告总数
+     */
+    private Integer report;
+
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/NavFailAnalysisVO.java

@@ -0,0 +1,29 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: NavFailAnalysisVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 13:39
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class NavFailAnalysisVO {
+    /**
+     * 无法从PDF文件中获取到数据
+     */
+    private Long pdfNoData;
+    /**
+     * 缺少净值日期
+     */
+    private Long priceDateMiss;
+    /**
+     * 单位净值和累计净值和资产净值均缺失
+     */
+    private Long navMiss;
+    /**
+     * 基金名称和备案编码均缺失
+     */
+    private Long fundNameNumberMiss;
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/ReportFailAnalysisVO.java

@@ -0,0 +1,31 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: NavFailAnalysisVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 13:39
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class ReportFailAnalysisVO {
+
+    /**
+     * 报告为扫描件
+     */
+    private Long scannedFile;
+    /**
+     * 报告不是基协统一格式
+     */
+    private Long errorAmacFileType;
+    /**
+     * 报告水印干扰导致部分没有解析
+     */
+    private Long watermarkFileError;
+    /**
+     * 报告不是定期报告
+     */
+    private Long noReport;
+
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/ValuationFailAnalysisVO.java

@@ -0,0 +1,33 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: NavFailAnalysisVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 13:39
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class ValuationFailAnalysisVO {
+    /**
+     * ●文件格式错误
+     */
+    private Long fileTypeError;
+    /**
+     * ●无市值列或无数量列
+     */
+    private Long columnMiss;
+    /**
+     *●非数值数据
+     */
+    private Long numbericMiss;
+    /**
+     * ●无数据
+     */
+    private Long noData;
+    /**
+     * ●模板不支持
+     */
+    private Long templateError;
+}

service-base/src/main/resources/mapper/AssetMapper.xml

@@ -69,5 +69,8 @@
                isvalid, creatorid, createtime, updaterid, updatetime
         from PPW_EMAIL.asset where isvalid=1 and fund_id=#{fundId}
     </select>
+    <select id="countAssetTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.asset where isvalid=1
+    </select>
 
 </mapper>

service-base/src/main/resources/mapper/DistributionMapper.xml

@@ -137,4 +137,7 @@
     <select id="getSumDistributeByFundId" resultType="java.math.BigDecimal" parameterType="java.lang.String">
         select sum(distribution) from PPW_EMAIL.distribution where isvalid =1 and fund_id=#{fundId} and distribute_date <![CDATA[ <= ]]> #{distributeDate}
     </select>
+    <select id="countDistributionTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.distribution where isvalid =1
+    </select>
 </mapper>

service-base/src/main/resources/mapper/EmailFundAssetMapper.xml

@@ -227,6 +227,9 @@
                isvalid, creatorid, createtime, updaterid, updatetime
         from PPW_EMAIL.email_fund_asset where isvalid=1 and fund_id=#{fundId}
     </select>
+    <select id="countEmailAssetTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.email_fund_asset where isvalid=1
+    </select>
 
     <delete id="deleteByFileId">
         update PPW_EMAIL.email_fund_asset

service-base/src/main/resources/mapper/EmailFundNavMapper.xml

@@ -272,6 +272,9 @@
         where nav.isvalid =1 and nav.exception_status=1
         and nav.fund_id = #{fundId}
     </select>
+    <select id="countEmailNavTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.email_fund_nav where isvalid=1
+    </select>
 
 
 </mapper>

service-base/src/main/resources/mapper/EmailFundNavService.xml

@@ -55,5 +55,8 @@
         where c.isvalid=1 and f.isvalid=1
         and f.fund_id=#{fundId}
     </select>
+    <select id="countCompanyTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.pvn_company_info where isvalid=1
+    </select>
 
 </mapper>

service-base/src/main/resources/mapper/EmailParseInfoMapper.xml

@@ -154,5 +154,66 @@
         #{fileId}
         </foreach>
     </select>
+    <select id="searchEmailDataBoard" resultType="java.util.Map"
+            parameterType="com.simuwang.base.pojo.dto.query.DataboardQuery">
+        select parse_status as "parse_status",count(id) as "total" from PPW_EMAIL.email_parse_info
+        where isvalid=1
+        <if test="startDate != null  and startDate != ''">
+            and email_date >= #{startDate}
+        </if>
+        <if test="endDate != null and endDate != ''">
+            and email_date <![CDATA[ <= ]]> #{endDate}
+        </if>
+        <if test="email !=null and email !=''">
+            and email like concat('%',#{email},'%')
+        </if>
+        <if test="emailType !=null">
+            and email_type = #{emailType}
+        </if>
+        group by parse_status
+    </select>
+    <select id="searchEmailTypeCount" resultType="java.util.Map"
+            parameterType="com.simuwang.base.pojo.dto.query.DataboardQuery">
+        select email_type as "email_type",count(id)  as "total" from PPW_EMAIL.email_parse_info
+        where isvalid=1
+        <if test="startDate != null  and startDate != ''">
+            and email_date >= #{startDate}
+        </if>
+        <if test="endDate != null and endDate != ''">
+            and email_date <![CDATA[ <= ]]> #{endDate}
+        </if>
+        <if test="email !=null and email !=''">
+            and email like concat('%',#{email},'%')
+        </if>
+        <if test="emailType !=null">
+            and email_type = #{emailType}
+        </if>
+        group by email_type
+    </select>
+    <select id="countpdfNoData" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.email_parse_info
+        where isvalid=1
+        <if test="item.startDate != null  and item.startDate != ''">
+            and email_date >= #{item.startDate}
+        </if>
+        <if test="item.endDate != null and item.endDate != ''">
+            and email_date <![CDATA[ <= ]]> #{item.endDate}
+        </if>
+        <if test="item.email !=null and item.email !=''">
+            and email like concat('%',#{item.email},'%')
+        </if>
+        <if test="item.emailType !=null">
+            and email_type = #{item.emailType}
+        </if>
+        <if test="errorInfo !=null">
+            and fail_reason like concat('%',#{errorInfo},'%')
+        </if>
+    </select>
+    <select id="countEmailTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.email_parse_info where isvalid=1
+        <if test="emailType !=null">
+            and email_type = #{emailType}
+        </if>
+    </select>
 
 </mapper>

service-base/src/main/resources/mapper/FundInfoMapper.xml

@@ -215,4 +215,7 @@
                   on info.issuer_id = ic.company_id and ic.isvalid=1
         where info.isvalid=1 and info.fund_id=#{fundId}
     </select>
+    <select id="countFundTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.pvn_fund_info where isvalid=1
+    </select>
 </mapper>

service-base/src/main/resources/mapper/NavMapper.xml

@@ -98,6 +98,9 @@
           and fund_id = #{fundId}
         order by price_date
     </select>
+    <select id="countNavTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.nav where isvalid=1
+    </select>
 
 
 </mapper>

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -23,6 +23,8 @@ import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
 import com.simuwang.base.pojo.dto.EmailFundNavDTO;
 import com.simuwang.base.pojo.dto.MailboxInfoDTO;
 import com.simuwang.base.pojo.dto.report.ParseResult;
+import com.simuwang.base.pojo.dto.query.DataboardQuery;
+import com.simuwang.base.pojo.dto.report.PythonResult;
 import com.simuwang.base.pojo.dto.report.ReportData;
 import com.simuwang.base.pojo.dto.report.ReportParseStatus;
 import com.simuwang.base.pojo.dto.report.ReportParserParams;
@@ -76,6 +78,8 @@ public class EmailParseService {
     private final ValuationTableMapper valuationTableMapper;
     private final ValuationTableAttributeMapper valuationTableAttributeMapper;
     private final FundPositionDetailMapper fundPositionDetailMapper;
+    private final DistributionMapper distributionMapper;
+    private final CompanyInformationMapper companyInformationMapper;
 
     @Value("${email.file.path}")
     private String path;
@@ -96,7 +100,7 @@ public class EmailParseService {
                              AssetMapper assetMapper, NavMapper navMapper, FundService fundService,
                              FundAliasMapper fundAliasMapper,
                              ValuationTableMapper valuationTableMapper, ValuationTableAttributeMapper valuationTableAttributeMapper,
-                             FundPositionDetailMapper fundPositionDetailMapper) {
+                             FundPositionDetailMapper fundPositionDetailMapper, DistributionMapper distributionMapper, CompanyInformationMapper companyInformationMapper) {
         this.emailTypeRuleMapper = emailTypeRuleMapper;
         this.emailRuleConfig = emailRuleConfig;
         this.emailFieldMapper = emailFieldMapper;
@@ -113,6 +117,8 @@ public class EmailParseService {
         this.valuationTableMapper = valuationTableMapper;
         this.valuationTableAttributeMapper = valuationTableAttributeMapper;
         this.fundPositionDetailMapper = fundPositionDetailMapper;
+        this.distributionMapper = distributionMapper;
+        this.companyInformationMapper = companyInformationMapper;
     }
 
     /**
@@ -181,7 +187,7 @@ public class EmailParseService {
         emailId = saveEmailParseInfo(emailParseInfoDO);
 
         // python 报告解析接口结果
-        List<ParseResult<ReportData>> dataList = ListUtil.list(false);
+        List<ReportData> dataList = ListUtil.list(false);
         for (Map.Entry<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavEntry : fileNameNavMap.entrySet()) {
             // 保存邮件文件表
             EmailContentInfoDTO emailContentInfoDTO = fileNameNavEntry.getKey();
@@ -199,10 +205,12 @@ public class EmailParseService {
             if (CollUtil.isEmpty(fundNavDTOList) && !Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) {
                 continue;
             }
-            if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) {
-                // 解析结果（可以从python获取或者自行解析）并保存报告
-                ParseResult<ReportData> parseResult = this.parseReportAndHandleResult(fileId, emailContentInfoDTO);
-                dataList.add(parseResult);
+            // python接口解析结果
+            ReportData data = this.requestPyAndResult(fileId, emailContentInfoDTO);
+            if (data != null) {
+                // 保存报告解析数据
+                this.reportWriterFactory.getInstance(data.getReportType()).write(data);
+                dataList.add(data);
             }
             for (EmailFundNavDTO fundNavDTO : fundNavDTOList) {
                 // 设置净值数据的解析状态
@@ -216,18 +224,17 @@ public class EmailParseService {
         // 更新邮件解析结果 -> 当【净值日期】和【备案编码/基金名称】能正常解读，即识别为【成功】
         long successNavCount = fileNameNavMap.values().stream().flatMap(List::stream).filter(e -> e != null && StrUtil.isBlank(e.getFailReason())).count();
         emailParseStatus = successNavCount >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
+        // 报告邮件有一条成功就表示整体成功
+        if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) && CollUtil.isNotEmpty(dataList)) {
+            long count = dataList.size();
+            emailParseStatus = count >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
+        }
         String failReason = null;
         if (emailParseStatus == EmailParseStatusConst.FAIL) {
             // 邮件解析失败时 -> 保存失败原因
             int hasPdfFile = emailContentInfoDTOList.stream().map(EmailContentInfoDTO::getFilePath).anyMatch(ExcelUtil::isPdf) ? 1 : 0;
             List<EmailFundNavDTO> navDTOList = fileNameNavMap.values().stream().flatMap(List::stream).toList();
-            failReason = hasPdfFile == 1 && CollUtil.isEmpty(navDTOList) ? "无法从PDF文件中获取到数据" : navDTOList.stream().map(EmailFundNavDTO::getFailReason).distinct().collect(Collectors.joining("/"));
-        }
-        // 报告邮件有一条失败就表示整个邮件解析失败
-        if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) && CollUtil.isNotEmpty(dataList)) {
-            failReason = dataList.stream().filter(e -> !Objects.equals(1, e.getStatus()))
-                    .findFirst().map(ParseResult::getMsg).orElse(null);
-            emailParseStatus = failReason != null ? EmailParseStatusConst.FAIL : EmailParseStatusConst.SUCCESS;
+            failReason = hasPdfFile == 1 && CollUtil.isEmpty(navDTOList) ? "无法从pdf文件中获取到数据" : navDTOList.stream().map(EmailFundNavDTO::getFailReason).distinct().collect(Collectors.joining("/"));
         }
         emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus, failReason);
     }
@@ -366,80 +373,61 @@ public class EmailParseService {
         }).collect(Collectors.toList());
     }
 
-    private ParseResult<ReportData> parseReportAndHandleResult(int fileId, EmailContentInfoDTO emailContentInfoDTO) {
-        ParseResult<ReportData> result = new ParseResult<>();
+    private ReportData requestPyAndResult(int fileId, EmailContentInfoDTO emailContentInfoDTO) {
         String fileName = emailContentInfoDTO.getFileName();
         Integer emailType = emailContentInfoDTO.getEmailType();
-        if (!Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) || StrUtil.isBlank(fileName)) {
-            result.setStatus(ReportParseStatus.NOT_A_REPORT.getCode());
-            result.setMsg(ReportParseStatus.NOT_A_REPORT.getMsg());
-            return result;
-        }
-        Pattern pattern = Pattern.compile("S(?:[A-Z]{0}[0-9]{5}|[A-Z][0-9]{4}|[A-Z]{2}[0-9]{3}|[A-Z]{3}[0-9]{2})");
-        Matcher matcher = pattern.matcher(fileName);
-        String registerNumber = null;
-        if (matcher.find()) {
-            registerNumber = matcher.group();
-        }
-        // 类型识别---先识别季度报告，没有季度再识别年度报告，最后识别月报
-        ReportType reportType = ReportType.MONTHLY;
-        if (StrUtil.containsAny(fileName, ReportType.QUARTERLY.getPatterns())) {
-            reportType = ReportType.QUARTERLY;
-        } else if (StrUtil.containsAny(fileName, ReportType.ANNUALLY.getPatterns())) {
-            reportType = ReportType.ANNUALLY;
-        }
-        // 解析器--如果开启python解析则直接调用python接口，否则根据文件后缀获取对应解析器
-        ReportParserFileType fileType;
-        if (Objects.equals(Boolean.TRUE, this.properties.getEnablePyParser())) {
-            fileType = ReportParserFileType.PYTHON;
-        } else {
-            String fileSuffix = StrUtil.subAfter(fileName, ".", true);
-            fileType = ReportParserFileType.getBySuffix(fileSuffix);
-        }
-        // 解析报告
-        ReportParserParams params = null;
         ReportData reportData = null;
-        StopWatch parserWatch = new StopWatch();
-        parserWatch.start();
-        try {
-            params = ReportParserParams.builder().fileId(fileId).filename(fileName)
-                    .filepath(emailContentInfoDTO.getFilePath()).registerNumber(registerNumber).build();
-            ReportParser<ReportData> instance = this.reportParserFactory.getInstance(reportType, fileType);
-            reportData = instance.parse(params);
-            result.setStatus(1);
-            result.setMsg("报告解析成功");
-            result.setData(reportData);
-        } catch (ReportParseException e) {
-            log.error("报告{}解析失败\n{}", params, e.getMsg());
-            result.setStatus(e.getCode());
-            result.setMsg(e.getMsg());
-        } catch (Exception e) {
-            log.error("报告{}解析失败\n{}", params, ExceptionUtil.stacktraceToString(e));
-            result.setStatus(ReportParseStatus.PARSE_FAIL.getCode());
-            result.setMsg(StrUtil.format(ReportParseStatus.PARSE_FAIL.getMsg(), e.getMessage()));
-        } finally {
-            parserWatch.stop();
-            if (log.isInfoEnabled()) {
-                log.info("报告{}解析结果为{}，耗时{}ms", params, reportData, parserWatch.getTotalTimeMillis());
+        if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) {
+            if (StrUtil.isBlank(fileName)) {
+                return null;
             }
-        }
-        // 保存报告解析结果
-        if (reportData != null) {
-            StopWatch writeWatch = new StopWatch();
-            writeWatch.start();
+            Pattern pattern = Pattern.compile("S(?:[A-Z]{0}[0-9]{5}|[A-Z][0-9]{4}|[A-Z]{2}[0-9]{3}|[A-Z]{3}[0-9]{2})");
+            Matcher matcher = pattern.matcher(fileName);
+            String registerNumber = null;
+            if (matcher.find()) {
+                registerNumber = matcher.group();
+            }
+            int type = 0;
+            if (fileName.contains("季报") || fileName.contains("季度")) {
+                type = 1;
+            } else if (fileName.contains("年报") || fileName.contains("年度")) {
+                type = 2;
+            }
+            String api = "/api/v1/parse/amac_report";
+            Map<String, Object> params = MapUtil.newHashMap(16);
+            params.put("file_id", fileId);
+            params.put("file_path", emailContentInfoDTO.getFilePath());
+            params.put("register_number", registerNumber);
+            params.put("file_type", type);
+            params.put("file_name", fileName);
+            if (StrUtil.isNotBlank(registerNumber)) {
+                FundAndCompanyInfoDO info = this.fundInfoMapper.queryFundAndTrustByRegisterNumber(registerNumber);
+                if (info != null) {
+                    params.put("fund_name", info.getFundName());
+                    params.put("trust_name", info.getCompanyName());
+                }
+            }
+            long millis = System.currentTimeMillis();
             try {
-                ReportWriter<ReportData> instance = this.reportWriterFactory.getInstance(reportType);
-                instance.write(reportData);
+                String body = HttpUtil.post(this.pyBaseUrl + api, JSONUtil.toJsonStr(params));
+                PythonResult<?> result = PythonReportConverter.convert(JSONUtil.parseObj(body), type);
+                if (!Objects.equals(1, result.getStatus())) {
+                    log.warn("报告{} 解析失败：{}", params, result.getMsg());
+                    return null;
+                }
+                reportData = result.getData();
+                if (log.isInfoEnabled()) {
+                    log.info("报告{}结果为:\n{}", params, reportData);
+                }
             } catch (Exception e) {
-                log.error("报告{}结果保存失败\n{}", params, ExceptionUtil.stacktraceToString(e));
+                log.error("请求python的报告解析接口报错\n{}", ExceptionUtil.stacktraceToString(e));
             } finally {
-                writeWatch.stop();
                 if (log.isInfoEnabled()) {
-                    log.info("报告{}解析结果保存完成，耗时{}ms", params, writeWatch.getTotalTimeMillis());
+                    log.info("当前报告{}解析完成，总计耗时{}ms", params, (System.currentTimeMillis() - millis));
                 }
             }
         }
-        return result;
+        return reportData;
     }
 
     private void saveNavAndAssetNet(Integer fileId, List<EmailFundNavDTO> fundNavDTOList, Date parseDate) {
@@ -853,8 +841,8 @@ public class EmailParseService {
                     emailContentInfoDTOList.add(emailContentInfoDTO);
                 }
                 if (CollUtil.isNotEmpty(emailContentInfoDTOList)) {
-                    // 估值表或定期报告邮件不展示正文html文件
-                    if (emailType.equals(EmailTypeConst.VALUATION_EMAIL_TYPE) || emailType.equals(EmailTypeConst.REPORT_EMAIL_TYPE)) {
+                    // 估值表邮件不展示正文html文件
+                    if (emailType.equals(EmailTypeConst.VALUATION_EMAIL_TYPE)) {
                         emailContentInfoDTOList = emailContentInfoDTOList.stream().filter(e -> !ExcelUtil.isHTML(e.getFilePath())).toList();
                     }
                     emailContentInfoDTOList.forEach(e -> {
@@ -903,6 +891,100 @@ public class EmailParseService {
         }
     }
 
+    public EmailParseCountBoardVO searchEmailCount(DataboardQuery databoardQuery) {
+        List<Map<String, Object>> dataList = emailParseInfoMapper.searchEmailDataBoard(databoardQuery);
+        EmailParseCountBoardVO result = new EmailParseCountBoardVO();
+        Integer total = 0;
+        for(Map<String, Object> data : dataList){
+            if(1 == ((Integer)data.get("parse_status")).intValue()){
+                result.setSuccess(((Long)data.get("total")).intValue());
+            }else{
+                result.setFail(((Long)data.get("total")).intValue());
+            }
+            total+=((Long)data.get("total")).intValue();
+        }
+        result.setTotal(total);
+        return result;
+    }
+
+    public EmailParseTypeBoardVO searchEmailTypeCount(DataboardQuery databoardQuery) {
+        //邮件类型，1-净值，2-估值表，3-定期报告
+        List<Map<String, Object>> dataList = emailParseInfoMapper.searchEmailTypeCount(databoardQuery);
+        EmailParseTypeBoardVO result = new EmailParseTypeBoardVO();
+        Integer total = 0;
+        for(Map<String, Object> data : dataList){
+            Integer emailType = (Integer) data.get("email_type");
+            Long totalType = (Long)data.get("total");
+            if(1 == emailType){
+                result.setNav(totalType.intValue());
+            }else if(2 == emailType){
+                result.setValuation(totalType.intValue());
+            }else{
+                result.setReport(totalType.intValue());
+            }
+            total+=totalType.intValue();
+        }
+        result.setTotal(total);
+        return result;
+    }
+
+    public EmailParseFailAnalysisVO parseFailAnalysis(DataboardQuery databoardQuery) {
+        EmailParseFailAnalysisVO emailParseFailAnalysisVO = new EmailParseFailAnalysisVO();
+        if(databoardQuery.getEmailType() == null || databoardQuery.getEmailType().equals(1)){
+            NavFailAnalysisVO navFailAnalysisVO = new NavFailAnalysisVO();
+            Long pdfNoData = emailParseInfoMapper.countpdfNoData(databoardQuery,"无法从PDF文件中获取到数据");
+            navFailAnalysisVO.setPdfNoData(pdfNoData);
+            Long priceDateMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"缺少净值日期");
+            navFailAnalysisVO.setPriceDateMiss(priceDateMiss);
+            Long navMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"单位净值和累计净值和资产净值均缺失");
+            navFailAnalysisVO.setNavMiss(navMiss);
+            Long fundNameNumberMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"单位净值和累计净值和资产净值均缺失");
+            navFailAnalysisVO.setFundNameNumberMiss(fundNameNumberMiss);
+            emailParseFailAnalysisVO.setNavFailAnalysisVO(navFailAnalysisVO);
+        }else if(databoardQuery.getEmailType().equals(2)){
+            ValuationFailAnalysisVO valuationFailAnalysisVO = new ValuationFailAnalysisVO();
+            Long fileTypeError = emailParseInfoMapper.countpdfNoData(databoardQuery,"文件格式错误");
+            valuationFailAnalysisVO.setFileTypeError(fileTypeError);
+            Long columnMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"无市值列或无数量列");
+            valuationFailAnalysisVO.setColumnMiss(columnMiss);
+            Long numbericMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"非数值数据");
+            valuationFailAnalysisVO.setNumbericMiss(numbericMiss);
+            Long noData = emailParseInfoMapper.countpdfNoData(databoardQuery,"无数据");
+            valuationFailAnalysisVO.setNoData(noData);
+            Long templateError = emailParseInfoMapper.countpdfNoData(databoardQuery,"模板不支持");
+            valuationFailAnalysisVO.setTemplateError(templateError);
+            emailParseFailAnalysisVO.setValuationFailAnalysisVO(valuationFailAnalysisVO);
+        }else if(databoardQuery.getEmailType().equals(3)){
+            ReportFailAnalysisVO reportFailAnalysisVO = new ReportFailAnalysisVO();
+            Long scannedFile = emailParseInfoMapper.countpdfNoData(databoardQuery,"报告为扫描件");
+            reportFailAnalysisVO.setScannedFile(scannedFile);
+            Long errorAmacFileType = emailParseInfoMapper.countpdfNoData(databoardQuery,"报告不是基协统一格式");
+            reportFailAnalysisVO.setErrorAmacFileType(errorAmacFileType);
+            Long watermarkFileError = emailParseInfoMapper.countpdfNoData(databoardQuery,"报告水印干扰导致部分没有解析");
+            reportFailAnalysisVO.setWatermarkFileError(watermarkFileError);
+            Long noReport = emailParseInfoMapper.countpdfNoData(databoardQuery,"报告不是定期报告");
+            reportFailAnalysisVO.setNoReport(noReport);
+            emailParseFailAnalysisVO.setReportFailAnalysisVO(reportFailAnalysisVO);
+        }
+        return emailParseFailAnalysisVO;
+    }
+
+    public EmailParseDataViewVO dataOverview(DataboardQuery databoardQuery) {
+        EmailParseDataViewVO dataViewVO = new EmailParseDataViewVO();
+        dataViewVO.setEmailNum(emailParseInfoMapper.countEmailTotal(null));
+        dataViewVO.setNavEmailNum(emailParseInfoMapper.countEmailTotal(1));
+        dataViewVO.setValuationEmailNum(emailParseInfoMapper.countEmailTotal(2));
+        dataViewVO.setReportEmailNum(emailParseInfoMapper.countEmailTotal(3));
+        dataViewVO.setParseNavNum(emailFundNavMapper.countEmailNavTotal());
+        dataViewVO.setParseAssetNum(emailFundAssetMapper.countEmailAssetTotal());
+        dataViewVO.setNavNum(navMapper.countNavTotal());
+        dataViewVO.setAssetNum(assetMapper.countAssetTotal());
+        dataViewVO.setDistribute(distributionMapper.countDistributionTotal());
+        dataViewVO.setFundNum(fundInfoMapper.countFundTotal());
+        dataViewVO.setCompanyNum(companyInformationMapper.countCompanyTotal());
+        return dataViewVO;
+    }
+
     private static class PythonData {
         private Integer fileId;
         private Integer status;

service-manage/src/main/java/com/simuwang/manage/api/databoard/DataboardController.java

@@ -0,0 +1,76 @@
+package com.simuwang.manage.api.databoard;
+
+import com.simuwang.base.pojo.dto.query.DataboardQuery;
+import com.simuwang.base.pojo.vo.EmailParseCountBoardVO;
+import com.simuwang.base.pojo.vo.EmailParseDataViewVO;
+import com.simuwang.base.pojo.vo.EmailParseFailAnalysisVO;
+import com.simuwang.base.pojo.vo.EmailParseTypeBoardVO;
+import com.simuwang.daq.service.EmailParseService;
+import com.simuwang.logging.SystemLog;
+import com.smppw.common.pojo.ResultVo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Map;
+
+/**
+ * FileName: 数据看板
+ * Author:   chenjianhua
+ * Date:     2024/10/14 10:33
+ * Description: ${DESCRIPTION}
+ */
+@SystemLog("数据看板")
+@RestController
+@RequestMapping("/v1/data-board")
+public class DataboardController {
+
+    @Autowired
+    private EmailParseService emailParseService;
+
+    /**
+     * 邮件总数
+     * @param databoardQuery
+     * @return
+     */
+    @SystemLog("邮件总数")
+    @RequestMapping("/email-count")
+    public ResultVo<EmailParseCountBoardVO> emailCount(DataboardQuery databoardQuery){
+        EmailParseCountBoardVO data = emailParseService.searchEmailCount(databoardQuery);
+        return new ResultVo<>(data);
+    }
+
+    /**
+     * 邮件类型分布
+     * @param databoardQuery
+     * @return
+     */
+    @RequestMapping("/email-type-count")
+    public ResultVo<EmailParseTypeBoardVO> emailTypeCount(DataboardQuery databoardQuery){
+        EmailParseTypeBoardVO data = emailParseService.searchEmailTypeCount(databoardQuery);
+        return new ResultVo<>(data);
+    }
+
+    /**
+     * 解析错误邮件分析
+     * @param databoardQuery
+     * @return
+     */
+    @RequestMapping("/parse-fail-analysis")
+    public ResultVo<EmailParseFailAnalysisVO> parseFailAnalysis(DataboardQuery databoardQuery){
+        EmailParseFailAnalysisVO data = emailParseService.parseFailAnalysis(databoardQuery);
+        return new ResultVo<>(data);
+    }
+
+    /**
+     * 数据概况
+     * @param databoardQuery
+     * @return
+     */
+    @RequestMapping("/data-overview")
+    public ResultVo<EmailParseDataViewVO> dataOverview(DataboardQuery databoardQuery){
+        EmailParseDataViewVO data = emailParseService.dataOverview(databoardQuery);
+        return new ResultVo<>(data);
+    }
+
+}