feat:excel报告解析逻辑完成

service-base/src/main/java/com/simuwang/base/common/enums/ReportType.java

@@ -4,9 +4,9 @@ import lombok.Getter;
 
 @Getter
 public enum ReportType {
-    MONTHLY(0, "月报", new String[]{"月", "月度", "月报"}),
-    QUARTERLY(1, "季报", new String[]{"季", "季度", "季报"}),
-    ANNUALLY(2, "年报", new String[]{"年", "年度", "年报"});
+    MONTHLY(0, "月", new String[]{"月", "月度", "月报"}),
+    QUARTERLY(1, "季", new String[]{"季", "季度", "季报"}),
+    ANNUALLY(2, "年", new String[]{"年", "年度", "年报"});
 
     private final int type;
     private final String label;

service-base/src/main/java/com/simuwang/base/mapper/AssetMapper.java

@@ -23,4 +23,6 @@ public interface AssetMapper {
     List<AssetDO> selectAssetByFundId(@Param("fundId")String fundId);
 
     void batchDeleteAsset(@Param("fundId")String sourceFundId,@Param("priceDateList") List<String> priceDateList);
+
+    Long countAssetTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/CompanyInformationMapper.java

@@ -17,4 +17,6 @@ public interface CompanyInformationMapper {
     public List<Map<String, String>> searchCompanyInfoByKeyword(@Param("keyword") String keyword);
 
     String getCompanyNameByFundId(@Param("fundId") String fundId);
+
+    Long countCompanyTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/DistributionMapper.java

@@ -37,4 +37,6 @@ public interface DistributionMapper {
     List<DistributionDO> getDistributionByFundId(@Param("fundId")String fundId,@Param("distributeType") DistributeType distributeType);
 
     BigDecimal getSumDistributeByFundId(@Param("fundId")String fundId,@Param("distributeDate") String distributeDate);
+
+    Long countDistributionTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailFundAssetMapper.java

@@ -29,4 +29,6 @@ public interface EmailFundAssetMapper {
     List<EmailParseDetailDO> selectFundAssetByFielId(@Param("fileId") Integer fileId, @Param("priceDate") String priceDate);
 
     List<EmailFundAssetDO> selectAssetByFundId(@Param("fundId") String sourceFundId);
+
+    Long countEmailAssetTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailFundNavMapper.java

@@ -33,4 +33,6 @@ public interface EmailFundNavMapper {
     List<EmailParseDetailDO> selectFundNavByFielId(@Param("fileId") Integer fileId, @Param("priceDate") String priceDate);
 
     List<EmailFundNavDO> selectNavByFundId(@Param("fundId") String sourceFundId);
+
+    Long countEmailNavTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/EmailParseInfoMapper.java

@@ -2,6 +2,7 @@ package com.simuwang.base.mapper;
 
 import com.simuwang.base.pojo.dos.EmailParseInfoDO;
 import com.simuwang.base.pojo.dto.EmailInfoDTO;
+import com.simuwang.base.pojo.dto.query.DataboardQuery;
 import com.simuwang.base.pojo.dto.query.EmailParseQuery;
 import com.simuwang.base.pojo.vo.EmailParseInfoVO;
 import org.apache.ibatis.annotations.Mapper;
@@ -9,6 +10,7 @@ import org.apache.ibatis.annotations.Param;
 
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
 @Mapper
 public interface EmailParseInfoMapper {
@@ -30,4 +32,12 @@ public interface EmailParseInfoMapper {
     void updateParseTime(@Param("id") Integer id, @Param("parseDate") Date parseDate);
 
     List<EmailInfoDTO> queryValuationEmailByFileId(@Param("fileIdList") List<Integer> fileIdList);
+
+    List<Map<String, Object>> searchEmailDataBoard(DataboardQuery databoardQuery);
+
+    List<Map<String, Object>> searchEmailTypeCount(DataboardQuery databoardQuery);
+
+    Long countpdfNoData(@Param("item") DataboardQuery databoardQuery, @Param("errorInfo")String errorInfo);
+
+    Long countEmailTotal(@Param("emailType")Integer emailType);
 }

service-base/src/main/java/com/simuwang/base/mapper/FundInfoMapper.java

@@ -49,4 +49,6 @@ public interface FundInfoMapper {
     String getInceptionDateByFundId(@Param("fundId")String fundId);
 
     FundInfoDO searchFundDetail(@Param("fundId")String fundId);
+
+    Long countFundTotal();
 }

service-base/src/main/java/com/simuwang/base/mapper/NavMapper.java

@@ -28,4 +28,6 @@ public interface NavMapper {
     List<NavDO> selectNavByFundId(@Param("fundId")String fundId);
 
     void batchDeleteNav(@Param("fundId")String sourceFundId, @Param("priceDateList")List<String> priceDateList);
+
+    Long countNavTotal();
 }

service-base/src/main/java/com/simuwang/base/pojo/dto/query/DataboardQuery.java

@@ -0,0 +1,58 @@
+package com.simuwang.base.pojo.dto.query;
+
+/**
+ * FileName: DataboardQuery
+ * Author:   chenjianhua
+ * Date:     2024/10/14 10:38
+ * Description: ${DESCRIPTION}
+ */
+public class DataboardQuery {
+    /**
+     * 邮箱地址
+     */
+    private String email;
+    /**
+     * 邮箱类型
+     */
+    private Integer emailType;
+    /**
+     * 开始时间
+     */
+    private String startDate;
+    /**
+     * 结束时间
+     */
+    private String endDate;
+
+    public String getEmail() {
+        return email;
+    }
+
+    public void setEmail(String email) {
+        this.email = email;
+    }
+
+    public Integer getEmailType() {
+        return emailType;
+    }
+
+    public void setEmailType(Integer emailType) {
+        this.emailType = emailType;
+    }
+
+    public String getStartDate() {
+        return startDate;
+    }
+
+    public void setStartDate(String startDate) {
+        this.startDate = startDate;
+    }
+
+    public String getEndDate() {
+        return endDate;
+    }
+
+    public void setEndDate(String endDate) {
+        this.endDate = endDate;
+    }
+}

service-base/src/main/java/com/simuwang/base/pojo/dto/report/BaseReportDTO.java

@@ -53,29 +53,29 @@ public abstract class BaseReportDTO<T extends BaseReportDO> implements Serializa
         try {
             // 日期格式化，支持三种格式：yyyy年MM月dd日、yyyy-MM-dd和yyyy/MM/dd
             return DateUtil.parse(input.trim(),
-                    DatePattern.CHINESE_DATE_PATTERN, DatePattern.NORM_DATE_PATTERN, "yyyy/MM/dd");
+                    DatePattern.CHINESE_DATE_PATTERN, DatePattern.NORM_DATE_PATTERN, "yyyy/MM/dd", "yyyyMMdd");
         } catch (Exception ignored) {
         }
         return null;
     }
 
     /**
-     * 字符串转数字
+     * 字符串转数字，如果数据没有或者转换失败则用0填充
      *
      * @param input 待转换的字符串
      * @return /
      */
     protected BigDecimal toBigDecimal(String input) {
         if (StrUtil.isBlank(input)) {
-            return null;
+            return BigDecimal.ZERO;
         }
         try {
-            // 替换掉数字分位的逗号
-            String cleanedInput = input.trim().replaceAll(",", "");
+            // 替换掉非正常的正负小数字符
+            String cleanedInput = input.trim().replaceAll("[^\\s" + "[-+]?\\d*\\.?\\d+" + "]", "");
             // 创建BigDecimal对象
             return new BigDecimal(cleanedInput);
         } catch (NumberFormatException ignored) {
         }
-        return null;
+        return BigDecimal.ZERO;
     }
 }

service-base/src/main/java/com/simuwang/base/pojo/dto/report/ReportParseStatus.java

@@ -14,6 +14,7 @@ public enum ReportParseStatus implements StatusCode {
     PARSE_FINANCIAL_INFO_FAIL(21012, "报告[{}] 没有解析到基金财务指标信息"),
     PARSE_INDUSTRY_INFO_FAIL(21013, "报告[{}] 没有解析到基金行业配置信息"),
     PARSE_ASSET_INFO_FAIL(21014, "报告[{}] 没有解析到基金资产配置信息"),
+    PARSE_SHARE_INFO_FAIL(21015, "报告[{}] 没有解析到基金份额变动信息"),
     ;
     private final int code;
     private final String msg;

service-base/src/main/java/com/simuwang/base/pojo/vo/EmailParseCountBoardVO.java

@@ -0,0 +1,20 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: EmailParseCountBoardVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 12:50
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class EmailParseCountBoardVO {
+
+    private Integer total;
+
+    private Integer success;
+
+    private Integer fail;
+
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/EmailParseDataViewVO.java

@@ -0,0 +1,58 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: EmailParseCountBoardVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 12:50
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class EmailParseDataViewVO {
+    /**
+     * 邮件数量
+     */
+    private Long emailNum;
+    /**
+     * 净值邮件数量
+     */
+    private Long navEmailNum;
+    /**
+     * 估值表邮件数量
+     */
+    private Long valuationEmailNum;
+    /**
+     * 定期报告邮件数量
+     */
+    private Long reportEmailNum;
+    /**
+     * 解析净值数量
+     */
+    private Long parseNavNum;
+    /**
+     * 解析规模数量
+     */
+    private Long parseAssetNum;
+    /**
+     * 入库净值数量
+     */
+    private Long NavNum;
+    /**
+     * 入库规模数量
+     */
+    private Long assetNum;
+    /**
+     * 分红数量
+     */
+    private Long distribute;
+    /**
+     * 基金数量
+     */
+    private Long fundNum;
+    /**
+     * 公司数量
+     */
+    private Long companyNum;
+
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/EmailParseFailAnalysisVO.java

@@ -0,0 +1,25 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: EmailParseFailAnalysisVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 13:39
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class EmailParseFailAnalysisVO {
+    /**
+     * 净值类型邮件返回结构
+     */
+    private NavFailAnalysisVO navFailAnalysisVO;
+    /**
+     * 估值表类型邮件返回结构
+     */
+    private ValuationFailAnalysisVO valuationFailAnalysisVO;
+    /**
+     * 报告类型邮件返回结构
+     */
+    private ReportFailAnalysisVO reportFailAnalysisVO;
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/EmailParseTypeBoardVO.java

@@ -0,0 +1,30 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: EmailParseCountBoardVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 12:50
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class EmailParseTypeBoardVO {
+    /**
+     * 总数
+     */
+    private Integer total;
+    /**
+     * 估值表总数
+     */
+    private Integer valuation;
+    /**
+     * 净值总数
+     */
+    private Integer nav;
+    /**
+     * 报告总数
+     */
+    private Integer report;
+
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/NavFailAnalysisVO.java

@@ -0,0 +1,29 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: NavFailAnalysisVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 13:39
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class NavFailAnalysisVO {
+    /**
+     * 无法从PDF文件中获取到数据
+     */
+    private Long pdfNoData;
+    /**
+     * 缺少净值日期
+     */
+    private Long priceDateMiss;
+    /**
+     * 单位净值和累计净值和资产净值均缺失
+     */
+    private Long navMiss;
+    /**
+     * 基金名称和备案编码均缺失
+     */
+    private Long fundNameNumberMiss;
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/ReportFailAnalysisVO.java

@@ -0,0 +1,31 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: NavFailAnalysisVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 13:39
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class ReportFailAnalysisVO {
+
+    /**
+     * 报告为扫描件
+     */
+    private Long scannedFile;
+    /**
+     * 报告不是基协统一格式
+     */
+    private Long errorAmacFileType;
+    /**
+     * 报告水印干扰导致部分没有解析
+     */
+    private Long watermarkFileError;
+    /**
+     * 报告不是定期报告
+     */
+    private Long noReport;
+
+}

service-base/src/main/java/com/simuwang/base/pojo/vo/ValuationFailAnalysisVO.java

@@ -0,0 +1,33 @@
+package com.simuwang.base.pojo.vo;
+
+import lombok.Data;
+
+/**
+ * FileName: NavFailAnalysisVO
+ * Author:   chenjianhua
+ * Date:     2024/10/14 13:39
+ * Description: ${DESCRIPTION}
+ */
+@Data
+public class ValuationFailAnalysisVO {
+    /**
+     * ●文件格式错误
+     */
+    private Long fileTypeError;
+    /**
+     * ●无市值列或无数量列
+     */
+    private Long columnMiss;
+    /**
+     *●非数值数据
+     */
+    private Long numbericMiss;
+    /**
+     * ●无数据
+     */
+    private Long noData;
+    /**
+     * ●模板不支持
+     */
+    private Long templateError;
+}

service-base/src/main/resources/mapper/AssetMapper.xml

@@ -69,5 +69,8 @@
                isvalid, creatorid, createtime, updaterid, updatetime
         from PPW_EMAIL.asset where isvalid=1 and fund_id=#{fundId}
     </select>
+    <select id="countAssetTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.asset where isvalid=1
+    </select>
 
 </mapper>

service-base/src/main/resources/mapper/DistributionMapper.xml

@@ -137,4 +137,7 @@
     <select id="getSumDistributeByFundId" resultType="java.math.BigDecimal" parameterType="java.lang.String">
         select sum(distribution) from PPW_EMAIL.distribution where isvalid =1 and fund_id=#{fundId} and distribute_date <![CDATA[ <= ]]> #{distributeDate}
     </select>
+    <select id="countDistributionTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.distribution where isvalid =1
+    </select>
 </mapper>

service-base/src/main/resources/mapper/EmailFundAssetMapper.xml

@@ -227,6 +227,9 @@
                isvalid, creatorid, createtime, updaterid, updatetime
         from PPW_EMAIL.email_fund_asset where isvalid=1 and fund_id=#{fundId}
     </select>
+    <select id="countEmailAssetTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.email_fund_asset where isvalid=1
+    </select>
 
     <delete id="deleteByFileId">
         update PPW_EMAIL.email_fund_asset

service-base/src/main/resources/mapper/EmailFundNavMapper.xml

@@ -272,6 +272,9 @@
         where nav.isvalid =1 and nav.exception_status=1
         and nav.fund_id = #{fundId}
     </select>
+    <select id="countEmailNavTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.email_fund_nav where isvalid=1
+    </select>
 
 
 </mapper>

service-base/src/main/resources/mapper/EmailFundNavService.xml

@@ -55,5 +55,8 @@
         where c.isvalid=1 and f.isvalid=1
         and f.fund_id=#{fundId}
     </select>
+    <select id="countCompanyTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.pvn_company_info where isvalid=1
+    </select>
 
 </mapper>

service-base/src/main/resources/mapper/EmailParseInfoMapper.xml

@@ -154,5 +154,66 @@
         #{fileId}
         </foreach>
     </select>
+    <select id="searchEmailDataBoard" resultType="java.util.Map"
+            parameterType="com.simuwang.base.pojo.dto.query.DataboardQuery">
+        select parse_status as "parse_status",count(id) as "total" from PPW_EMAIL.email_parse_info
+        where isvalid=1
+        <if test="startDate != null  and startDate != ''">
+            and email_date >= #{startDate}
+        </if>
+        <if test="endDate != null and endDate != ''">
+            and email_date <![CDATA[ <= ]]> #{endDate}
+        </if>
+        <if test="email !=null and email !=''">
+            and email like concat('%',#{email},'%')
+        </if>
+        <if test="emailType !=null">
+            and email_type = #{emailType}
+        </if>
+        group by parse_status
+    </select>
+    <select id="searchEmailTypeCount" resultType="java.util.Map"
+            parameterType="com.simuwang.base.pojo.dto.query.DataboardQuery">
+        select email_type as "email_type",count(id)  as "total" from PPW_EMAIL.email_parse_info
+        where isvalid=1
+        <if test="startDate != null  and startDate != ''">
+            and email_date >= #{startDate}
+        </if>
+        <if test="endDate != null and endDate != ''">
+            and email_date <![CDATA[ <= ]]> #{endDate}
+        </if>
+        <if test="email !=null and email !=''">
+            and email like concat('%',#{email},'%')
+        </if>
+        <if test="emailType !=null">
+            and email_type = #{emailType}
+        </if>
+        group by email_type
+    </select>
+    <select id="countpdfNoData" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.email_parse_info
+        where isvalid=1
+        <if test="item.startDate != null  and item.startDate != ''">
+            and email_date >= #{item.startDate}
+        </if>
+        <if test="item.endDate != null and item.endDate != ''">
+            and email_date <![CDATA[ <= ]]> #{item.endDate}
+        </if>
+        <if test="item.email !=null and item.email !=''">
+            and email like concat('%',#{item.email},'%')
+        </if>
+        <if test="item.emailType !=null">
+            and email_type = #{item.emailType}
+        </if>
+        <if test="errorInfo !=null">
+            and fail_reason like concat('%',#{errorInfo},'%')
+        </if>
+    </select>
+    <select id="countEmailTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.email_parse_info where isvalid=1
+        <if test="emailType !=null">
+            and email_type = #{emailType}
+        </if>
+    </select>
 
 </mapper>

service-base/src/main/resources/mapper/FundInfoMapper.xml

@@ -215,4 +215,7 @@
                   on info.issuer_id = ic.company_id and ic.isvalid=1
         where info.isvalid=1 and info.fund_id=#{fundId}
     </select>
+    <select id="countFundTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.pvn_fund_info where isvalid=1
+    </select>
 </mapper>

service-base/src/main/resources/mapper/NavMapper.xml

@@ -98,6 +98,9 @@
           and fund_id = #{fundId}
         order by price_date
     </select>
+    <select id="countNavTotal" resultType="java.lang.Long">
+        select count(1) from PPW_EMAIL.nav where isvalid=1
+    </select>
 
 
 </mapper>

service-daq/src/main/java/com/simuwang/daq/components/CustomExcelMultiSheetListener.java

@@ -61,7 +61,13 @@ public class CustomExcelMultiSheetListener extends AnalysisEventListener<LinkedH
             List<String> tableRow = ListUtil.list(true);
             int colCount = this.table.getColCount() <= 0 ? row.size() : this.table.getColCount() + this.customExcelTable.getStartCol();
             for (int i = this.customExcelTable.getStartCol(); i < colCount; i++) {
-                tableRow.add(ReportParseUtils.cleaningValue(row.get(i)));
+                String cell = ReportParseUtils.cleaningValue(row.get(i), false);
+                if (cell != null) {
+                    cell = cell.replace(" ", "")
+                            .replace("（", "(").replace("）", ")")
+                            .replace("：", ":").replace("；", ";");
+                }
+                tableRow.add(cell);
             }
             this.table.addRow(tableRow);
         }

service-daq/src/main/java/com/simuwang/daq/components/ReportParseUtils.java

@@ -1,10 +1,12 @@
 package com.simuwang.daq.components;
 
 import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.util.StrUtil;
 
 import java.util.Calendar;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -23,6 +25,10 @@ public final class ReportParseUtils {
      * 主要财务指标识别列名称
      */
     public static final List<String> FINANCIAL_INDICATORS_COLUMN_NAMES = ListUtil.list(false);
+    /**
+     * 资产配置明细和大类关系映射
+     */
+    public static final Map<String, String> ASSET_ALLOCATION_TYPE_MAPPER = MapUtil.newHashMap(32, true);
 
     static {
         // 财务指标
@@ -76,6 +82,59 @@ public final class ReportParseUtils {
         SHARE_CHANGE_COLUMN_NAMES.add("期末基金总份额/期末基金实缴总额");
         SHARE_CHANGE_COLUMN_NAMES.add("报告期期间基金拆分变动份额");
         SHARE_CHANGE_COLUMN_NAMES.add("报告期期间基金总申购份额");
+
+        // 资产配置
+        ASSET_ALLOCATION_TYPE_MAPPER.put("银行存款", "现金类资产");
+        // 境内未上市、未挂牌公司股权投资
+        ASSET_ALLOCATION_TYPE_MAPPER.put("股权投资", "境内未上市、未挂牌公司股权投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其中:优先股", "境内未上市、未挂牌公司股权投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其他股权类投资", "境内未上市、未挂牌公司股权投资");
+        // 上市公司定向增发投资
+        ASSET_ALLOCATION_TYPE_MAPPER.put("上市公司定向增发投资", "上市公司定向增发投资");
+        // 新三板投资
+        ASSET_ALLOCATION_TYPE_MAPPER.put("新三板挂牌企业投资", "新三板投资");
+        // 境内证券投资规模
+        ASSET_ALLOCATION_TYPE_MAPPER.put("结算备付金", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("存出保证金", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("股票投资", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("债券投资", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其中:银行间市场债券", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其中:利率债", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其中:信用债", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("资产支持证券", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("基金投资(公募基金)", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其中:货币基金", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("期货及衍生品交易保证金", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("买入返售金融资产", "境内证券投资规模");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其他证券类标的", "境内证券投资规模");
+        // 资管计划投资
+        ASSET_ALLOCATION_TYPE_MAPPER.put("商业银行理财产品投资", "资管计划投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("信托计划投资", "资管计划投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("基金公司及其子公司资产管理计划投资", "资管计划投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("保险资产管理计划投资", "资管计划投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("证券公司及其子公司资产管理计划投资", "资管计划投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("期货公司及其子公司资产管理计划投资", "资管计划投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("私募基金产品投资", "资管计划投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("未在协会备案的合伙企业份额", "资管计划投资");
+        // 另类投资
+        ASSET_ALLOCATION_TYPE_MAPPER.put("另类投资", "另类投资");
+        // 境内债权类投资
+        ASSET_ALLOCATION_TYPE_MAPPER.put("银行委托贷款规模", "境内债权类投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("信托贷款", "境内债权类投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("应收账款投资", "境内债权类投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("各类受(收)益权投资", "境内债权类投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("票据(承兑汇票等)投资", "境内债权类投资");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其他债权投资", "境内债权类投资");
+        // 境外投资
+        ASSET_ALLOCATION_TYPE_MAPPER.put("境外投资", "境外投资");
+        // 其他资产
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其他资产", "其他资产");
+        // 基金负债情况
+        ASSET_ALLOCATION_TYPE_MAPPER.put("债券回购总额", "基金负债情况");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("融资、融券总额", "基金负债情况");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其中:融券总额", "基金负债情况");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("银行借款总额", "基金负债情况");
+        ASSET_ALLOCATION_TYPE_MAPPER.put("其他融资总额", "基金负债情况");
     }
 
     /**

service-daq/src/main/java/com/simuwang/daq/components/report/parser/excel/ExcelAnnuallyReportParser.java

@@ -0,0 +1,240 @@
+package com.simuwang.daq.components.report.parser.excel;
+
+import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapUtil;
+import cn.hutool.core.util.StrUtil;
+import com.simuwang.base.common.exception.ReportParseException;
+import com.simuwang.base.mapper.EmailFieldMappingMapper;
+import com.simuwang.base.pojo.dto.report.*;
+import com.simuwang.daq.components.ReportParseUtils;
+import com.simuwang.daq.components.report.parser.ReportParserConstant;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+@Component(ReportParserConstant.PARSER_EXCEL_ANNUALLY)
+public class ExcelAnnuallyReportParser extends AbstractExcelReportParser<AnnuallyReportData> {
+    public ExcelAnnuallyReportParser(EmailFieldMappingMapper fieldMappingMapper) {
+        super(fieldMappingMapper);
+    }
+
+    @Override
+    public String getParser() {
+        return ReportParserConstant.PARSER_EXCEL_ANNUALLY;
+    }
+
+    @Override
+    protected List<CustomExcelTable> customExcelTables() {
+        List<CustomExcelTable> customExcelTables = ListUtil.list(true);
+        customExcelTables.add(new CustomExcelTable("fundInfo", "基金基本情况", 2));
+        customExcelTables.add(new CustomExcelTable("fundInfo", "基金产品说明", 2, 4));
+        customExcelTables.add(new CustomExcelTable("financialIndicators", "主要会计数据和财务指标", 4));
+        customExcelTables.add(new CustomExcelTable("financialIndicators", "级基金主要会计数据和财务指标", 4, 10));
+        customExcelTables.add(new CustomExcelTable("assetAllocation", "期末基金资产组合情况", 3));
+        customExcelTables.add(new CustomExcelTable("investmentIndustry", "报告期末按行业分类的股票投资组合", 4));
+        customExcelTables.add(new CustomExcelTable("investmentIndustry", "报告期末按行业分类的港股通投资股票投资组合", 3));
+        customExcelTables.add(new CustomExcelTable("shareChange", "基金份额变动情况", 2, 6));
+        customExcelTables.add(new CustomExcelTable("shareChange", "级基金份额变动情况", 2, 6));
+        return customExcelTables;
+    }
+
+    @Override
+    protected AnnuallyReportData parseExtInfoAndSetData(ReportBaseInfoDTO reportInfo, ReportFundInfoDTO reportFundInfo, List<SimpleTable> tables) {
+        Integer fileId = reportInfo.getFileId();
+        String reportName = reportInfo.getReportName();
+        // 主要财务指标
+        List<ReportFinancialIndicatorsDTO> financialIndicators = this.buildFinancialIndicatorsInfo(fileId, reportName, tables);
+        // 资产配置
+        List<ReportAssetAllocationDTO> assetAllocations = this.buildAssetAllocationInfo(fileId, reportName, tables);
+        // 行业配置
+        List<ReportInvestmentIndustryDTO> investmentIndustries = this.buildInvestmentIndustryInfo(fileId, reportName, tables);
+        // 份额变动
+        List<ReportShareChangeDTO> shareChanges = this.buildShareChangeInfo(fileId, reportName, tables);
+        // 构建返回结构
+        AnnuallyReportData reportData = new AnnuallyReportData(reportInfo, reportFundInfo);
+        reportData.setFinancialIndicators(financialIndicators);
+        reportData.setAssetAllocation(assetAllocations);
+        reportData.setInvestmentIndustry(investmentIndustries);
+        reportData.setShareChange(shareChanges);
+        return reportData;
+    }
+
+    @Override
+    protected ReportFundInfoDTO buildFundInfo(ReportParserParams params, List<SimpleTable> tables) {
+        List<SimpleTable> simpleTables = tables.stream().filter(e -> "fundInfo".equals(e.getTableKey())).collect(Collectors.toList());
+        if (CollUtil.isEmpty(simpleTables)) {
+            throw new ReportParseException(ReportParseStatus.PARSE_FUND_INFO_FAIL, params.getFilename());
+        }
+        Map<String, Object> fundInfoMap = MapUtil.newHashMap(32);
+        for (SimpleTable table : simpleTables) {
+            Map<String, Object> baseInfoMap = MapUtil.newHashMap(32);
+            for (int i = 0; i < table.getTables().size(); i++) {
+                List<String> cols = table.getTables().get(i);
+                for (int j = 0; j < 1; j++) {
+                    baseInfoMap.put(cols.get(j), cols.get(j + 1));
+                }
+            }
+            fundInfoMap.putAll(baseInfoMap);
+        }
+        ReportFundInfoDTO info = new ReportFundInfoDTO(params.getFileId());
+        this.buildInfo(fundInfoMap, info);
+        return info;
+    }
+
+    /**
+     * 构建基金份额变动信息
+     *
+     * @param fileId   文件id
+     * @param filename 文件名称
+     * @param tables   所有表格
+     * @return /
+     */
+    private List<ReportShareChangeDTO> buildShareChangeInfo(Integer fileId, String filename, List<SimpleTable> tables) {
+        List<SimpleTable> simpleTables = tables.stream().filter(e -> "shareChange".equals(e.getTableKey())).collect(Collectors.toList());
+        if (CollUtil.isEmpty(simpleTables)) {
+            throw new ReportParseException(ReportParseStatus.PARSE_SHARE_INFO_FAIL, filename);
+        }
+        Function<SimpleTable, Map<String, Object>> function = t -> {
+            Map<String, Object> infoMap = MapUtil.newHashMap(16);
+            for (List<String> table : t.getTables()) {
+                String name = table.get(0);
+                if (name == null || !ReportParseUtils.SHARE_CHANGE_COLUMN_NAMES.contains(name)) {
+                    continue;
+                }
+                infoMap.put(name, table.get(1));
+            }
+            return infoMap;
+        };
+        return this.buildLevelDto(fileId, simpleTables, ReportShareChangeDTO.class, function);
+    }
+
+    /**
+     * 构建基金主要财务指标信息
+     *
+     * @param fileId   文件id
+     * @param filename 报告名称
+     * @param tables   所有表格
+     * @return /
+     */
+    private List<ReportFinancialIndicatorsDTO> buildFinancialIndicatorsInfo(Integer fileId, String filename, List<SimpleTable> tables) {
+        List<SimpleTable> simpleTables = tables.stream().filter(e -> "financialIndicators".equals(e.getTableKey())).collect(Collectors.toList());
+        if (CollUtil.isEmpty(simpleTables)) {
+            throw new ReportParseException(ReportParseStatus.PARSE_FINANCIAL_INFO_FAIL, filename);
+        }
+        List<ReportFinancialIndicatorsDTO> dtos = ListUtil.list(false);
+        // 分级基金
+        String titles = simpleTables.stream().map(SimpleTable::getTitle).collect(Collectors.joining(","));
+        List<String> levels = ReportParseUtils.matchTieredFund(titles);
+        // 存在分级基金
+        for (int k = 0; k < simpleTables.size(); k++) {
+            SimpleTable table = simpleTables.get(k);
+            int colCount = table.getColCount();
+            for (int j = 1; j < colCount; j++) {
+                Map<String, Object> infoMap = MapUtil.newHashMap(16);
+                String year = ReportParseUtils.cleaningValue(table.getCell(0, j));
+                infoMap.put("年度", year);
+                for (int i = 0; i < table.getRowCount(); i++) {
+                    String columnName = ReportParseUtils.cleaningValue(table.getCell(i, 0));
+                    if (!CollUtil.contains(ReportParseUtils.FINANCIAL_INDICATORS_COLUMN_NAMES, columnName)) {
+                        continue;
+                    }
+                    String value = ReportParseUtils.cleaningValue(table.getCell(i, j));
+                    infoMap.put(columnName, value);
+                }
+                ReportFinancialIndicatorsDTO dto = new ReportFinancialIndicatorsDTO(fileId);
+                this.buildInfo(infoMap, dto);
+                dto.setLevel(levels.get(k));
+                dtos.add(dto);
+            }
+        }
+        return dtos;
+    }
+
+    /**
+     * 构建基金行业配置信息
+     *
+     * @param fileId   文件id
+     * @param filename 报告名称
+     * @param tables   所有表格
+     * @return /
+     */
+    private List<ReportInvestmentIndustryDTO> buildInvestmentIndustryInfo(Integer fileId, String filename, List<SimpleTable> tables) {
+        List<SimpleTable> simpleTables = tables.stream().filter(e -> "investmentIndustry".equals(e.getTableKey())).collect(Collectors.toList());
+        if (CollUtil.isEmpty(simpleTables)) {
+            throw new ReportParseException(ReportParseStatus.PARSE_INDUSTRY_INFO_FAIL, filename);
+        }
+        List<ReportInvestmentIndustryDTO> dtos = ListUtil.list(false);
+        for (SimpleTable table : simpleTables) {
+            int colCount = table.getColCount();
+            // 投资地区: 1-境内, 2-港股通
+            int investType = colCount == 4 ? 1 : 2;
+            int j = colCount == 4 ? 1 : 0;
+            // 按行遍历
+            for (int i = 0; i < table.getRowCount(); i++) {
+                String industryName = ReportParseUtils.cleaningValue(table.getCell(i, j));
+                if (StrUtil.isBlank(industryName) || !ReportParseUtils.INDUSTRY_COLUMN_NAMES.contains(industryName)) {
+                    continue;
+                }
+                ReportInvestmentIndustryDTO dto = new ReportInvestmentIndustryDTO(fileId);
+                dto.setInvestType(investType);
+                dto.setIndustryName(industryName);
+                dto.setMarketValue(ReportParseUtils.cleaningValue(table.getCell(i, j + 1)));
+                dto.setRatio(ReportParseUtils.cleaningValue(table.getCell(i, j + 2)));
+                dtos.add(dto);
+            }
+        }
+        return dtos;
+    }
+
+    /**
+     * 构建基金资产配置信息
+     *
+     * @param fileId   文件id
+     * @param filename 报告名称
+     * @param tables   所有表格
+     * @return /
+     */
+    private List<ReportAssetAllocationDTO> buildAssetAllocationInfo(Integer fileId, String filename, List<SimpleTable> tables) {
+        SimpleTable assetAllocationTable = tables.stream().filter(e -> "assetAllocation".equals(e.getTableKey())).findFirst().orElse(null);
+        if (assetAllocationTable == null) {
+            throw new ReportParseException(ReportParseStatus.PARSE_ASSET_INFO_FAIL, filename);
+        }
+        List<ReportAssetAllocationDTO> dtos = ListUtil.list(false);
+        // 按行遍历
+        for (List<String> row : assetAllocationTable.getTables()) {
+            String marketValueAndRemark = row.get(2);
+            String detail = row.get(1);
+            if (!ReportParseUtils.ASSET_ALLOCATION_TYPE_MAPPER.containsKey(detail)) {
+                continue;
+            }
+            String assetType = ReportParseUtils.ASSET_ALLOCATION_TYPE_MAPPER.get(detail);
+            if (StrUtil.contains(marketValueAndRemark, "#")) {
+                // 有#表示有备注，而且可能有多个，多个用分号分隔的.
+                List<String> marketValueAndRemarks = StrUtil.split(marketValueAndRemark, ";");
+                for (String mr : marketValueAndRemarks) {
+                    if (StrUtil.isBlank(mr)) {
+                        continue;
+                    }
+                    List<String> mrs = StrUtil.split(mr, "#");
+                    ReportAssetAllocationDTO dto = new ReportAssetAllocationDTO(fileId);
+                    dto.setAssetType(assetType);
+                    dto.setAssetDetails(detail);
+                    dto.setMarketValue(mrs.get(1));
+                    dto.setRemark(mrs.get(0));
+                    dtos.add(dto);
+                }
+            } else {
+                ReportAssetAllocationDTO dto = new ReportAssetAllocationDTO(fileId);
+                dto.setAssetType(assetType);
+                dto.setAssetDetails(detail);
+                dto.setMarketValue(marketValueAndRemark);
+                dtos.add(dto);
+            }
+        }
+        return dtos;
+    }
+}

service-daq/src/main/java/com/simuwang/daq/components/report/parser/excel/ExcelMonthlyReportParser.java

@@ -19,6 +19,11 @@ public class ExcelMonthlyReportParser extends AbstractExcelReportParser<MonthlyR
     }
 
     @Override
+    public String getParser() {
+        return ReportParserConstant.PARSER_EXCEL_MONTHLY;
+    }
+
+    @Override
     protected List<CustomExcelTable> customExcelTables() {
         List<CustomExcelTable> customExcelTables = ListUtil.list(true);
         customExcelTables.add(new CustomExcelTable("fundInfo", "基金概况", 4));

service-daq/src/main/java/com/simuwang/daq/components/report/parser/excel/ExcelQuarterlyReportParser.java

@@ -1,15 +1,19 @@
 package com.simuwang.daq.components.report.parser.excel;
 
+import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.collection.ListUtil;
 import cn.hutool.core.map.MapUtil;
+import cn.hutool.core.util.StrUtil;
 import com.simuwang.base.common.exception.ReportParseException;
 import com.simuwang.base.mapper.EmailFieldMappingMapper;
 import com.simuwang.base.pojo.dto.report.*;
+import com.simuwang.daq.components.ReportParseUtils;
 import com.simuwang.daq.components.report.parser.ReportParserConstant;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 @Component(ReportParserConstant.PARSER_EXCEL_QUARTERLY)
@@ -19,6 +23,11 @@ public class ExcelQuarterlyReportParser extends AbstractExcelReportParser<Quarte
     }
 
     @Override
+    public String getParser() {
+        return ReportParserConstant.PARSER_EXCEL_QUARTERLY;
+    }
+
+    @Override
     protected List<CustomExcelTable> customExcelTables() {
         List<CustomExcelTable> customExcelTables = ListUtil.list(true);
         customExcelTables.add(new CustomExcelTable("fundInfo", "基金基本情况", 2));
@@ -27,8 +36,8 @@ public class ExcelQuarterlyReportParser extends AbstractExcelReportParser<Quarte
         customExcelTables.add(new CustomExcelTable("assetAllocation", "期末基金资产组合情况", 3));
         customExcelTables.add(new CustomExcelTable("investmentIndustry", "报告期末按行业分类的股票投资组合", 4));
         customExcelTables.add(new CustomExcelTable("investmentIndustry", "报告期末按行业分类的港股通投资股票投资组合", 3));
-        customExcelTables.add(new CustomExcelTable("shareChange", "基金份额变动情况", 3, 6));
-        customExcelTables.add(new CustomExcelTable("shareChange", "级基金份额变动情况", 3, 6));
+        customExcelTables.add(new CustomExcelTable("shareChange", "基金份额变动情况", 2, 6));
+        customExcelTables.add(new CustomExcelTable("shareChange", "级基金份额变动情况", 2, 6));
         return customExcelTables;
     }
 
@@ -37,13 +46,13 @@ public class ExcelQuarterlyReportParser extends AbstractExcelReportParser<Quarte
         Integer fileId = reportInfo.getFileId();
         String reportName = reportInfo.getReportName();
         // 主要财务指标
-        List<ReportFinancialIndicatorsDTO> financialIndicators = this.buildFinancialIndicatorsInfo(fileId, tables);
+        List<ReportFinancialIndicatorsDTO> financialIndicators = this.buildFinancialIndicatorsInfo(fileId, reportName, tables);
         // 资产配置
         List<ReportAssetAllocationDTO> assetAllocations = this.buildAssetAllocationInfo(fileId, reportName, tables);
         // 行业配置
-        List<ReportInvestmentIndustryDTO> investmentIndustries = this.buildInvestmentIndustryInfo(fileId, tables);
+        List<ReportInvestmentIndustryDTO> investmentIndustries = this.buildInvestmentIndustryInfo(fileId, reportName, tables);
         // 份额变动
-        List<ReportShareChangeDTO> shareChanges = this.buildShareChangeInfo(fileId, tables);
+        List<ReportShareChangeDTO> shareChanges = this.buildShareChangeInfo(fileId, reportName, tables);
         // 构建返回结构
         QuarterlyReportData reportData = new QuarterlyReportData(reportInfo, reportFundInfo);
         reportData.setFinancialIndicators(financialIndicators);
@@ -72,26 +81,141 @@ public class ExcelQuarterlyReportParser extends AbstractExcelReportParser<Quarte
         return dto;
     }
 
-    private List<ReportShareChangeDTO> buildShareChangeInfo(Integer fileId, List<SimpleTable> tables) {
+    /**
+     * 构建基金份额变动信息
+     *
+     * @param fileId   文件id
+     * @param filename 文件名称
+     * @param tables   所有表格
+     * @return /
+     */
+    private List<ReportShareChangeDTO> buildShareChangeInfo(Integer fileId, String filename, List<SimpleTable> tables) {
         List<SimpleTable> simpleTables = tables.stream().filter(e -> "shareChange".equals(e.getTableKey())).collect(Collectors.toList());
-        return null;
+        if (CollUtil.isEmpty(simpleTables)) {
+            throw new ReportParseException(ReportParseStatus.PARSE_SHARE_INFO_FAIL, filename);
+        }
+        Function<SimpleTable, Map<String, Object>> function = t -> {
+            Map<String, Object> infoMap = MapUtil.newHashMap(16);
+            for (List<String> table : t.getTables()) {
+                String name = table.get(0);
+                if (name == null || !ReportParseUtils.SHARE_CHANGE_COLUMN_NAMES.contains(name)) {
+                    continue;
+                }
+                infoMap.put(name, table.get(1));
+            }
+            return infoMap;
+        };
+        return this.buildLevelDto(fileId, simpleTables, ReportShareChangeDTO.class, function);
     }
 
-    private List<ReportFinancialIndicatorsDTO> buildFinancialIndicatorsInfo(Integer fileId, List<SimpleTable> tables) {
+    /**
+     * 构建基金主要财务指标信息
+     *
+     * @param fileId   文件id
+     * @param filename 报告名称
+     * @param tables   所有表格
+     * @return /
+     */
+    private List<ReportFinancialIndicatorsDTO> buildFinancialIndicatorsInfo(Integer fileId, String filename, List<SimpleTable> tables) {
         List<SimpleTable> simpleTables = tables.stream().filter(e -> "financialIndicators".equals(e.getTableKey())).collect(Collectors.toList());
-        return null;
+        if (CollUtil.isEmpty(simpleTables)) {
+            throw new ReportParseException(ReportParseStatus.PARSE_FINANCIAL_INFO_FAIL, filename);
+        }
+        Function<SimpleTable, Map<String, Object>> function = t -> {
+            Map<String, Object> infoMap = MapUtil.newHashMap(16);
+            for (List<String> table : t.getTables()) {
+                String name = table.get(0);
+                if (name == null || !ReportParseUtils.FINANCIAL_INDICATORS_COLUMN_NAMES.contains(name)) {
+                    continue;
+                }
+                infoMap.put(name, table.get(1));
+            }
+            return infoMap;
+        };
+        return this.buildLevelDto(fileId, simpleTables, ReportFinancialIndicatorsDTO.class, function);
     }
 
-    private List<ReportInvestmentIndustryDTO> buildInvestmentIndustryInfo(Integer fileId, List<SimpleTable> tables) {
+    /**
+     * 构建基金行业配置信息
+     *
+     * @param fileId   文件id
+     * @param filename 报告名称
+     * @param tables   所有表格
+     * @return /
+     */
+    private List<ReportInvestmentIndustryDTO> buildInvestmentIndustryInfo(Integer fileId, String filename, List<SimpleTable> tables) {
         List<SimpleTable> simpleTables = tables.stream().filter(e -> "investmentIndustry".equals(e.getTableKey())).collect(Collectors.toList());
-        return null;
+        if (CollUtil.isEmpty(simpleTables)) {
+            throw new ReportParseException(ReportParseStatus.PARSE_INDUSTRY_INFO_FAIL, filename);
+        }
+        List<ReportInvestmentIndustryDTO> dtos = ListUtil.list(false);
+        for (SimpleTable table : simpleTables) {
+            int colCount = table.getColCount();
+            // 投资地区: 1-境内, 2-港股通
+            int investType = colCount == 4 ? 1 : 2;
+            int j = colCount == 4 ? 1 : 0;
+            // 按行遍历
+            for (int i = 0; i < table.getRowCount(); i++) {
+                String industryName = ReportParseUtils.cleaningValue(table.getCell(i, j));
+                if (StrUtil.isBlank(industryName) || !ReportParseUtils.INDUSTRY_COLUMN_NAMES.contains(industryName)) {
+                    continue;
+                }
+                ReportInvestmentIndustryDTO dto = new ReportInvestmentIndustryDTO(fileId);
+                dto.setInvestType(investType);
+                dto.setIndustryName(industryName);
+                dto.setMarketValue(ReportParseUtils.cleaningValue(table.getCell(i, j + 1)));
+                dto.setRatio(ReportParseUtils.cleaningValue(table.getCell(i, j + 2)));
+                dtos.add(dto);
+            }
+        }
+        return dtos;
     }
 
+    /**
+     * 构建基金资产配置信息
+     *
+     * @param fileId   文件id
+     * @param filename 报告名称
+     * @param tables   所有表格
+     * @return /
+     */
     private List<ReportAssetAllocationDTO> buildAssetAllocationInfo(Integer fileId, String filename, List<SimpleTable> tables) {
         SimpleTable assetAllocationTable = tables.stream().filter(e -> "assetAllocation".equals(e.getTableKey())).findFirst().orElse(null);
         if (assetAllocationTable == null) {
             throw new ReportParseException(ReportParseStatus.PARSE_ASSET_INFO_FAIL, filename);
         }
-        return null;
+        List<ReportAssetAllocationDTO> dtos = ListUtil.list(false);
+        // 按行遍历
+        for (List<String> row : assetAllocationTable.getTables()) {
+            String marketValueAndRemark = row.get(2);
+            String detail = row.get(1);
+            if (!ReportParseUtils.ASSET_ALLOCATION_TYPE_MAPPER.containsKey(detail)) {
+                continue;
+            }
+            String assetType = ReportParseUtils.ASSET_ALLOCATION_TYPE_MAPPER.get(detail);
+            if (StrUtil.contains(marketValueAndRemark, "#")) {
+                // 有#表示有备注，而且可能有多个，多个用分号分隔的.
+                List<String> marketValueAndRemarks = StrUtil.split(marketValueAndRemark, ";");
+                for (String mr : marketValueAndRemarks) {
+                    if (StrUtil.isBlank(mr)) {
+                        continue;
+                    }
+                    List<String> mrs = StrUtil.split(mr, "#");
+                    ReportAssetAllocationDTO dto = new ReportAssetAllocationDTO(fileId);
+                    dto.setAssetType(assetType);
+                    dto.setAssetDetails(detail);
+                    dto.setMarketValue(mrs.get(1));
+                    dto.setRemark(mrs.get(0));
+                    dtos.add(dto);
+                }
+            } else {
+                ReportAssetAllocationDTO dto = new ReportAssetAllocationDTO(fileId);
+                dto.setAssetType(assetType);
+                dto.setAssetDetails(detail);
+                dto.setMarketValue(marketValueAndRemark);
+                dtos.add(dto);
+            }
+        }
+        return dtos;
     }
 }

service-daq/src/main/java/com/simuwang/daq/components/report/parser/pdf/PDAnnuallyReportParser.java

@@ -12,6 +12,7 @@ import technology.tabula.Table;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 /**
@@ -72,7 +73,8 @@ public class PDAnnuallyReportParser extends PDQuarterlyReportParser<AnnuallyRepo
                 }
                 // 资产配置表格识别（兼容跨页的表格）获取表格中第二列的所有文字，判断所有文字中包含"股权投资"等字符串
                 texts = this.getTableColTexts(table, 1);
-                if (CollUtil.containsAny(texts, ListUtil.of("股权投资", "股票投资", "债券投资", "另类投资", "其他资产", "其他融资总额"))) {
+                Set<String> keys = ReportParseUtils.ASSET_ALLOCATION_TYPE_MAPPER.keySet();
+                if (CollUtil.containsAny(texts, keys)) {
                     this.assetAllocationTables.add(table);
                 }
             }

service-daq/src/main/java/com/simuwang/daq/components/report/parser/pdf/PDQuarterlyReportParser.java

@@ -13,10 +13,7 @@ import technology.tabula.RectangularTextContainer;
 import technology.tabula.Table;
 
 import java.awt.geom.Rectangle2D;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 import java.util.function.Function;
 
 /**
@@ -72,7 +69,11 @@ public class PDQuarterlyReportParser<T extends QuarterlyReportData> extends Abst
                 if (CollUtil.containsAny(texts, ReportParseUtils.INDUSTRY_COLUMN_NAMES)) {
                     this.investmentIndustryTables.add(table);
                 } else {
-                    this.assetAllocationTables.add(table);
+                    texts = this.getTableColTexts(table, 1);
+                    Set<String> keys = ReportParseUtils.ASSET_ALLOCATION_TYPE_MAPPER.keySet();
+                    if (CollUtil.containsAny(texts, keys)) {
+                        this.assetAllocationTables.add(table);
+                    }
                 }
             }
         }
@@ -179,7 +180,7 @@ public class PDQuarterlyReportParser<T extends QuarterlyReportData> extends Abst
                     continue;
                 }
                 String industryName = ReportParseUtils.cleaningValue(table.getCell(i, j).getText());
-                if (StrUtil.isBlank(industryName) || Objects.equals("合计", industryName)) {
+                if (StrUtil.isBlank(industryName) || !ReportParseUtils.INDUSTRY_COLUMN_NAMES.contains(industryName)) {
                     continue;
                 }
                 ReportInvestmentIndustryDTO dto = new ReportInvestmentIndustryDTO(fileId);
@@ -201,24 +202,20 @@ public class PDQuarterlyReportParser<T extends QuarterlyReportData> extends Abst
      */
     private List<ReportAssetAllocationDTO> buildAssetAllocationInfo(Integer fileId) {
         List<ReportAssetAllocationDTO> dtos = ListUtil.list(false);
-        String assetType = null;
         for (Table table : this.assetAllocationTables) {
             // 按行遍历
             for (@SuppressWarnings("all") List<RectangularTextContainer> row : table.getRows()) {
                 // x坐标升序（防止部分行乱序问题）
                 row.sort(Comparator.comparing(Rectangle2D.Float::getX));
-                // 大类
-                String type = ReportParseUtils.cleaningValue(row.get(0).getText());
-                if (StrUtil.isNotBlank(type)) {
-                    assetType = type;
-                }
                 // 金额、市值，有时是 “备注#金额”的格式
                 String marketValueAndRemark = ReportParseUtils.cleaningValue(row.get(2).getText());
-                if (StrUtil.isBlank(marketValueAndRemark) || StrUtil.isBlank(assetType)) {
-                    continue;
-                }
                 // 资产明细
                 String detail = ReportParseUtils.cleaningValue(row.get(1).getText(), false);
+                if (!ReportParseUtils.ASSET_ALLOCATION_TYPE_MAPPER.containsKey(detail)) {
+                    continue;
+                }
+                // 大类
+                String assetType = ReportParseUtils.ASSET_ALLOCATION_TYPE_MAPPER.get(detail);
                 if (StrUtil.contains(marketValueAndRemark, "#")) {
                     // 有#表示有备注，而且可能有多个，多个用分号分隔的.
                     List<String> marketValueAndRemarks = StrUtil.split(marketValueAndRemark, ";");

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -22,11 +22,13 @@ import com.simuwang.base.pojo.dos.*;
 import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
 import com.simuwang.base.pojo.dto.EmailFundNavDTO;
 import com.simuwang.base.pojo.dto.MailboxInfoDTO;
+import com.simuwang.base.pojo.dto.query.DataboardQuery;
 import com.simuwang.base.pojo.dto.report.ParseResult;
 import com.simuwang.base.pojo.dto.report.ReportData;
 import com.simuwang.base.pojo.dto.report.ReportParseStatus;
 import com.simuwang.base.pojo.dto.report.ReportParserParams;
 import com.simuwang.base.pojo.valuation.CmValuationTableAttribute;
+import com.simuwang.base.pojo.vo.*;
 import com.simuwang.daq.components.report.parser.ReportParser;
 import com.simuwang.daq.components.report.parser.ReportParserFactory;
 import com.simuwang.daq.components.report.writer.ReportWriter;
@@ -76,6 +78,10 @@ public class EmailParseService {
     private final ValuationTableMapper valuationTableMapper;
     private final ValuationTableAttributeMapper valuationTableAttributeMapper;
     private final FundPositionDetailMapper fundPositionDetailMapper;
+    private final DistributionMapper distributionMapper;
+    private final CompanyInformationMapper companyInformationMapper;
+    @Autowired
+    private FundInfoMapper fundInfoMapper;
 
     @Value("${email.file.path}")
     private String path;
@@ -96,7 +102,7 @@ public class EmailParseService {
                              AssetMapper assetMapper, NavMapper navMapper, FundService fundService,
                              FundAliasMapper fundAliasMapper,
                              ValuationTableMapper valuationTableMapper, ValuationTableAttributeMapper valuationTableAttributeMapper,
-                             FundPositionDetailMapper fundPositionDetailMapper) {
+                             FundPositionDetailMapper fundPositionDetailMapper, DistributionMapper distributionMapper, CompanyInformationMapper companyInformationMapper) {
         this.emailTypeRuleMapper = emailTypeRuleMapper;
         this.emailRuleConfig = emailRuleConfig;
         this.emailFieldMapper = emailFieldMapper;
@@ -113,6 +119,8 @@ public class EmailParseService {
         this.valuationTableMapper = valuationTableMapper;
         this.valuationTableAttributeMapper = valuationTableAttributeMapper;
         this.fundPositionDetailMapper = fundPositionDetailMapper;
+        this.distributionMapper = distributionMapper;
+        this.companyInformationMapper = companyInformationMapper;
     }
 
     /**
@@ -892,7 +900,99 @@ public class EmailParseService {
         }
         return null;
     }
+    public EmailParseCountBoardVO searchEmailCount(DataboardQuery databoardQuery) {
+        List<Map<String, Object>> dataList = emailParseInfoMapper.searchEmailDataBoard(databoardQuery);
+        EmailParseCountBoardVO result = new EmailParseCountBoardVO();
+        Integer total = 0;
+        for(Map<String, Object> data : dataList){
+            if(1 == ((Integer)data.get("parse_status")).intValue()){
+                result.setSuccess(((Long)data.get("total")).intValue());
+            }else{
+                result.setFail(((Long)data.get("total")).intValue());
+            }
+            total+=((Long)data.get("total")).intValue();
+        }
+        result.setTotal(total);
+        return result;
+    }
+
+    public EmailParseTypeBoardVO searchEmailTypeCount(DataboardQuery databoardQuery) {
+        //邮件类型，1-净值，2-估值表，3-定期报告
+        List<Map<String, Object>> dataList = emailParseInfoMapper.searchEmailTypeCount(databoardQuery);
+        EmailParseTypeBoardVO result = new EmailParseTypeBoardVO();
+        Integer total = 0;
+        for(Map<String, Object> data : dataList){
+            Integer emailType = (Integer) data.get("email_type");
+            Long totalType = (Long)data.get("total");
+            if(1 == emailType){
+                result.setNav(totalType.intValue());
+            }else if(2 == emailType){
+                result.setValuation(totalType.intValue());
+            }else{
+                result.setReport(totalType.intValue());
+            }
+            total+=totalType.intValue();
+        }
+        result.setTotal(total);
+        return result;
+    }
 
+    public EmailParseFailAnalysisVO parseFailAnalysis(DataboardQuery databoardQuery) {
+        EmailParseFailAnalysisVO emailParseFailAnalysisVO = new EmailParseFailAnalysisVO();
+        if(databoardQuery.getEmailType() == null || databoardQuery.getEmailType().equals(1)){
+            NavFailAnalysisVO navFailAnalysisVO = new NavFailAnalysisVO();
+            Long pdfNoData = emailParseInfoMapper.countpdfNoData(databoardQuery,"无法从PDF文件中获取到数据");
+            navFailAnalysisVO.setPdfNoData(pdfNoData);
+            Long priceDateMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"缺少净值日期");
+            navFailAnalysisVO.setPriceDateMiss(priceDateMiss);
+            Long navMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"单位净值和累计净值和资产净值均缺失");
+            navFailAnalysisVO.setNavMiss(navMiss);
+            Long fundNameNumberMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"单位净值和累计净值和资产净值均缺失");
+            navFailAnalysisVO.setFundNameNumberMiss(fundNameNumberMiss);
+            emailParseFailAnalysisVO.setNavFailAnalysisVO(navFailAnalysisVO);
+        }else if(databoardQuery.getEmailType().equals(2)){
+            ValuationFailAnalysisVO valuationFailAnalysisVO = new ValuationFailAnalysisVO();
+            Long fileTypeError = emailParseInfoMapper.countpdfNoData(databoardQuery,"文件格式错误");
+            valuationFailAnalysisVO.setFileTypeError(fileTypeError);
+            Long columnMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"无市值列或无数量列");
+            valuationFailAnalysisVO.setColumnMiss(columnMiss);
+            Long numbericMiss = emailParseInfoMapper.countpdfNoData(databoardQuery,"非数值数据");
+            valuationFailAnalysisVO.setNumbericMiss(numbericMiss);
+            Long noData = emailParseInfoMapper.countpdfNoData(databoardQuery,"无数据");
+            valuationFailAnalysisVO.setNoData(noData);
+            Long templateError = emailParseInfoMapper.countpdfNoData(databoardQuery,"模板不支持");
+            valuationFailAnalysisVO.setTemplateError(templateError);
+            emailParseFailAnalysisVO.setValuationFailAnalysisVO(valuationFailAnalysisVO);
+        }else if(databoardQuery.getEmailType().equals(3)){
+            ReportFailAnalysisVO reportFailAnalysisVO = new ReportFailAnalysisVO();
+            Long scannedFile = emailParseInfoMapper.countpdfNoData(databoardQuery,"报告为扫描件");
+            reportFailAnalysisVO.setScannedFile(scannedFile);
+            Long errorAmacFileType = emailParseInfoMapper.countpdfNoData(databoardQuery,"报告不是基协统一格式");
+            reportFailAnalysisVO.setErrorAmacFileType(errorAmacFileType);
+            Long watermarkFileError = emailParseInfoMapper.countpdfNoData(databoardQuery,"报告水印干扰导致部分没有解析");
+            reportFailAnalysisVO.setWatermarkFileError(watermarkFileError);
+            Long noReport = emailParseInfoMapper.countpdfNoData(databoardQuery,"报告不是定期报告");
+            reportFailAnalysisVO.setNoReport(noReport);
+            emailParseFailAnalysisVO.setReportFailAnalysisVO(reportFailAnalysisVO);
+        }
+        return emailParseFailAnalysisVO;
+    }
+
+    public EmailParseDataViewVO dataOverview(DataboardQuery databoardQuery) {
+        EmailParseDataViewVO dataViewVO = new EmailParseDataViewVO();
+        dataViewVO.setEmailNum(emailParseInfoMapper.countEmailTotal(null));
+        dataViewVO.setNavEmailNum(emailParseInfoMapper.countEmailTotal(1));
+        dataViewVO.setValuationEmailNum(emailParseInfoMapper.countEmailTotal(2));
+        dataViewVO.setReportEmailNum(emailParseInfoMapper.countEmailTotal(3));
+        dataViewVO.setParseNavNum(emailFundNavMapper.countEmailNavTotal());
+        dataViewVO.setParseAssetNum(emailFundAssetMapper.countEmailAssetTotal());
+        dataViewVO.setNavNum(navMapper.countNavTotal());
+        dataViewVO.setAssetNum(assetMapper.countAssetTotal());
+        dataViewVO.setDistribute(distributionMapper.countDistributionTotal());
+        dataViewVO.setFundNum(fundInfoMapper.countFundTotal());
+        dataViewVO.setCompanyNum(companyInformationMapper.countCompanyTotal());
+        return dataViewVO;
+    }
     private Message[] getEmailMessage(Folder folder, String protocol, Date startDate) {
         try {
             if (protocol.contains("imap")) {

service-daq/src/main/java/com/simuwang/daq/utils/ExcelReportParseUtil.java

@@ -10,13 +10,18 @@ import com.simuwang.daq.components.CustomExcelMultiSheetListener;
 import java.util.List;
 
 public class ExcelReportParseUtil {
-    public static final String filepath = "D:\\Documents\\workspace\\idea\\smppw\\data-daq\\service-daq\\src\\main\\java\\com\\simuwang\\daq\\utils\\ST8112_瑞富兴康稳健成长一号私募证券投资基金_信息披露月报.xlsx";
+    public static final String filepath = "D:\\Documents\\workspace\\idea\\smppw\\data-daq\\service-daq\\src\\main\\java\\com\\simuwang\\daq\\utils\\SEW698_旭日盛德价值成长一期私募证券投资基金_2023年第4季度报告.xlsx";
 
     public static void main(String[] args) {
         List<CustomExcelTable> customExcelTables = ListUtil.list(true);
-        customExcelTables.add(new CustomExcelTable("fundInfo", "基金概况", 4));
-        customExcelTables.add(new CustomExcelTable("netReport", "级基金净值表", 5, 2));
-        customExcelTables.add(new CustomExcelTable("netReport", "净值月报", 5, 2));
+        customExcelTables.add(new CustomExcelTable("fundInfo", "基金基本情况", 2));
+        customExcelTables.add(new CustomExcelTable("financialIndicators", "主要财务指标", 5, 6));
+        customExcelTables.add(new CustomExcelTable("financialIndicators", "级基金主要财务指标", 5, 6));
+        customExcelTables.add(new CustomExcelTable("assetAllocation", "期末基金资产组合情况", 3));
+        customExcelTables.add(new CustomExcelTable("investmentIndustry", "报告期末按行业分类的股票投资组合", 4));
+        customExcelTables.add(new CustomExcelTable("investmentIndustry", "报告期末按行业分类的港股通投资股票投资组合", 3));
+        customExcelTables.add(new CustomExcelTable("shareChange", "基金份额变动情况", 3, 6));
+        customExcelTables.add(new CustomExcelTable("shareChange", "级基金份额变动情况", 3, 6));
 
         CustomExcelMultiSheetListener readListener = new CustomExcelMultiSheetListener();
         ExcelReaderBuilder readerBuilder = EasyExcel.read(filepath);

service-deploy/src/test/java/com/simuwang/ApplicationTest.java

@@ -45,8 +45,8 @@ public class ApplicationTest {
     @Test
     public void reportTest() {
         MailboxInfoDTO emailInfoDTO = this.buildMailbox();
-        Date startDate = DateUtil.parse("2024-10-12 17:42:30", DateConst.YYYY_MM_DD_HH_MM_SS);
-        Date endDate = DateUtil.parse("2024-10-12 17:59:30", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date startDate = DateUtil.parse("2024-10-12 17:40:30", DateConst.YYYY_MM_DD_HH_MM_SS);
+        Date endDate = DateUtil.parse("2024-10-12 17:50:30", DateConst.YYYY_MM_DD_HH_MM_SS);
         try {
             emailParseService.parseEmail(emailInfoDTO, startDate, endDate);
         } catch (Exception e) {

service-manage/src/main/java/com/simuwang/manage/api/databoard/DataboardController.java

@@ -0,0 +1,76 @@
+package com.simuwang.manage.api.databoard;
+
+import com.simuwang.base.pojo.dto.query.DataboardQuery;
+import com.simuwang.base.pojo.vo.EmailParseCountBoardVO;
+import com.simuwang.base.pojo.vo.EmailParseDataViewVO;
+import com.simuwang.base.pojo.vo.EmailParseFailAnalysisVO;
+import com.simuwang.base.pojo.vo.EmailParseTypeBoardVO;
+import com.simuwang.daq.service.EmailParseService;
+import com.simuwang.logging.SystemLog;
+import com.smppw.common.pojo.ResultVo;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.Map;
+
+/**
+ * 数据看板
+ * Author:   chenjianhua
+ * Date:     2024/10/14 10:33
+ * Description: ${DESCRIPTION}
+ */
+@SystemLog("数据看板")
+@RestController
+@RequestMapping("/v1/data-board")
+public class DataboardController {
+
+    @Autowired
+    private EmailParseService emailParseService;
+
+    /**
+     * 邮件总数
+     * @param databoardQuery
+     * @return
+     */
+    @SystemLog("邮件总数")
+    @RequestMapping("/email-count")
+    public ResultVo<EmailParseCountBoardVO> emailCount(DataboardQuery databoardQuery){
+        EmailParseCountBoardVO data = emailParseService.searchEmailCount(databoardQuery);
+        return new ResultVo<>(data);
+    }
+
+    /**
+     * 邮件类型分布
+     * @param databoardQuery
+     * @return
+     */
+    @RequestMapping("/email-type-count")
+    public ResultVo<EmailParseTypeBoardVO> emailTypeCount(DataboardQuery databoardQuery){
+        EmailParseTypeBoardVO data = emailParseService.searchEmailTypeCount(databoardQuery);
+        return new ResultVo<>(data);
+    }
+
+    /**
+     * 解析错误邮件分析
+     * @param databoardQuery
+     * @return
+     */
+    @RequestMapping("/parse-fail-analysis")
+    public ResultVo<EmailParseFailAnalysisVO> parseFailAnalysis(DataboardQuery databoardQuery){
+        EmailParseFailAnalysisVO data = emailParseService.parseFailAnalysis(databoardQuery);
+        return new ResultVo<>(data);
+    }
+
+    /**
+     * 数据概况
+     * @param databoardQuery
+     * @return
+     */
+    @RequestMapping("/data-overview")
+    public ResultVo<EmailParseDataViewVO> dataOverview(DataboardQuery databoardQuery){
+        EmailParseDataViewVO data = emailParseService.dataOverview(databoardQuery);
+        return new ResultVo<>(data);
+    }
+
+}