data-daq/service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

package com.simuwang.daq.service;

import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.exceptions.ExceptionUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import com.simuwang.base.common.conts.*;
import com.simuwang.base.common.enums.ReportParserFileType;
import com.simuwang.base.common.enums.ReportType;
import com.simuwang.base.common.exception.ReportParseException;
import com.simuwang.base.common.util.*;
import com.simuwang.base.config.DaqProperties;
import com.simuwang.base.config.EmailRuleConfig;
import com.simuwang.base.mapper.daq.*;
import com.simuwang.base.pojo.dos.*;
import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
import com.simuwang.base.pojo.dto.EmailFundNavDTO;
import com.simuwang.base.pojo.dto.MailboxInfoDTO;
import com.simuwang.base.pojo.dto.query.DataboardQuery;
import com.simuwang.base.pojo.dto.report.*;
import com.simuwang.base.pojo.valuation.CmValuationTableAttribute;
import com.simuwang.base.pojo.vo.*;
import com.simuwang.daq.components.report.parser.ReportParser;
import com.simuwang.daq.components.report.parser.ReportParserFactory;
import com.simuwang.daq.components.report.writer.ReportWriter;
import com.simuwang.daq.components.report.writer.ReportWriterFactory;
import jakarta.mail.*;
import jakarta.mail.internet.MimeMessage;
import jakarta.mail.internet.MimeMultipart;
import jakarta.mail.search.ComparisonTerm;
import jakarta.mail.search.ReceivedDateTerm;
import jakarta.mail.search.SearchTerm;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch;

import java.io.File;
import java.math.BigDecimal;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * @author mozuwen
 * @date 2024-09-04
 * @description 邮件解析服务
 */
@Service
public class EmailParseService {

    public static final int stepSize = 10000;
    private static final Logger log = LoggerFactory.getLogger(EmailParseService.class);
    private final EmailTypeRuleMapper emailTypeRuleMapper;
    private final EmailRuleConfig emailRuleConfig;
    private final EmailFieldMappingMapper emailFieldMapper;
    private final EmailParserFactory emailParserFactory;
    private final EmailParseInfoMapper emailParseInfoMapper;
    private final EmailFileInfoMapper emailFileInfoMapper;
    private final EmailFundNavMapper emailFundNavMapper;
    private final EmailFundAssetMapper emailFundAssetMapper;
    private final EmailFundNavCollectionMapper emailFundNavCollectionMapper;
    private final AssetMapper assetMapper;
    private final NavMapper navMapper;
    private final FundService fundService;
    private final FundAliasMapper fundAliasMapper;
    private final ValuationTableMapper valuationTableMapper;
    private final ValuationTableAttributeMapper valuationTableAttributeMapper;
    private final FundPositionDetailMapper fundPositionDetailMapper;
    private final DistributionMapper distributionMapper;
    private final CompanyInformationMapper companyInformationMapper;
    private final FileManageMapper fileManageMapper;
    private FundInfoMapper fundInfoMapper;

    @Value("${email.file.path}")
    private String path;

    @Autowired
    private DaqProperties properties;

    /* 报告解析和入库的方法 */
    @Autowired
    private ReportParserFactory reportParserFactory;
    @Autowired
    private ReportWriterFactory reportWriterFactory;

    public EmailParseService(EmailTypeRuleMapper emailTypeRuleMapper, EmailRuleConfig emailRuleConfig,
                             EmailFieldMappingMapper emailFieldMapper, EmailParserFactory emailParserFactory,
                             EmailParseInfoMapper emailParseInfoMapper, EmailFileInfoMapper emailFileInfoMapper,
                             EmailFundNavMapper emailFundNavMapper, EmailFundAssetMapper emailFundAssetMapper, EmailFundNavCollectionMapper emailFundNavCollectionMapper,
                             AssetMapper assetMapper, NavMapper navMapper, FundService fundService,
                             FundAliasMapper fundAliasMapper,
                             ValuationTableMapper valuationTableMapper, ValuationTableAttributeMapper valuationTableAttributeMapper,
                             FundPositionDetailMapper fundPositionDetailMapper, DistributionMapper distributionMapper, CompanyInformationMapper companyInformationMapper, FileManageMapper fileManageMapper, FundInfoMapper fundInfoMapper) {
        this.emailTypeRuleMapper = emailTypeRuleMapper;
        this.emailRuleConfig = emailRuleConfig;
        this.emailFieldMapper = emailFieldMapper;
        this.emailParserFactory = emailParserFactory;
        this.emailParseInfoMapper = emailParseInfoMapper;
        this.emailFileInfoMapper = emailFileInfoMapper;
        this.emailFundNavMapper = emailFundNavMapper;
        this.emailFundAssetMapper = emailFundAssetMapper;
        this.emailFundNavCollectionMapper = emailFundNavCollectionMapper;
        this.assetMapper = assetMapper;
        this.navMapper = navMapper;
        this.fundService = fundService;
        this.fundAliasMapper = fundAliasMapper;

        this.valuationTableMapper = valuationTableMapper;
        this.valuationTableAttributeMapper = valuationTableAttributeMapper;
        this.fundPositionDetailMapper = fundPositionDetailMapper;
        this.distributionMapper = distributionMapper;
        this.companyInformationMapper = companyInformationMapper;
        this.fileManageMapper = fileManageMapper;
        this.fundInfoMapper = fundInfoMapper;
    }

    /**
     * 解析指定邮箱指定时间范围内的邮件
     *
     * @param mailboxInfoDTO 邮箱配置信息
     * @param startDate      邮件起始日期(yyyy-MM-dd HH:mm:ss)
     * @param endDate        邮件截止日期(yyyy-MM-dd HH:mm:ss, 为null,将解析邮件日期小于等于startDate的当天邮件)
     */
    public void parseEmail(MailboxInfoDTO mailboxInfoDTO, Date startDate, Date endDate) {
        log.info("开始邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
        // 邮件类型配置
        Map<Integer, List<String>> emailTypeMap = getEmailType();
        // 邮件字段识别映射表
        Map<String, List<String>> emailFieldMap = getEmailFieldMapping();
        Map<String, List<EmailContentInfoDTO>> emailContentMap;
        try {
            emailContentMap = realEmail(mailboxInfoDTO, emailTypeMap, startDate, endDate);
        } catch (Exception e) {
            log.info("采集邮件失败 -> 邮箱配置信息:{},堆栈信息:{}", mailboxInfoDTO, ExceptionUtil.stacktraceToString(e));
            return;
        }
        if (MapUtil.isEmpty(emailContentMap)) {
            log.info("未采集到邮件 -> 邮箱配置信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
            return;
        }
        for (Map.Entry<String, List<EmailContentInfoDTO>> emailEntry : emailContentMap.entrySet()) {
            List<EmailContentInfoDTO> emailContentInfoDTOList = emailEntry.getValue();
            if (CollUtil.isEmpty(emailContentInfoDTOList)) {
                log.warn("未采集到正文或附件");
                continue;
            }
            log.info("开始解析邮件数据 -> 邮件主题:{},邮件日期:{}", emailContentInfoDTOList.get(0).getEmailTitle(), emailContentInfoDTOList.get(0).getEmailDate());
            List<EmailFundNavDTO> emailFundNavDTOList = CollUtil.newArrayList();
            Map<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavMap = MapUtil.newHashMap();
            for (EmailContentInfoDTO emailContentInfoDTO : emailContentInfoDTOList) {
                try {
                    List<EmailFundNavDTO> fundNavDTOList = parseEmail(emailContentInfoDTO, emailFieldMap);
                    fileNameNavMap.put(emailContentInfoDTO, fundNavDTOList);
                    emailFundNavDTOList.addAll(fundNavDTOList);
                } catch (Exception e) {
                    log.error("堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
                }
            }
            // 保存相关信息 -> 邮件信息表,邮件文件表,邮件净值表,邮件规模表,基金净值表
            saveRelatedTable(mailboxInfoDTO.getAccount(), emailContentInfoDTOList, fileNameNavMap);
            log.info("结束邮件解析 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, DateUtil.format(startDate, DateConst.YYYY_MM_DD_HH_MM_SS), DateUtil.format(endDate, DateConst.YYYY_MM_DD_HH_MM_SS));
        }
    }

    public List<EmailFundNavDTO> parseEmail(EmailContentInfoDTO emailContentInfoDTO, Map<String, List<String>> emailFieldMap) {
        Integer emailType = emailContentInfoDTO.getEmailType();
        AbstractEmailParser emailParser = emailParserFactory.getInstance(emailType);
        //目前分红合同，基金合同，变更公告不做解析
        if(emailParser == null){
            return new ArrayList<>();
        }
        List<EmailFundNavDTO> emailFundNavDTOList = emailParser.parse(emailContentInfoDTO, emailFieldMap);
        //因为有些邮件是净值邮件，但标题是估值表的标题，会导致无法解析到数据，这里做个补丁如果估值表类型的邮件解析不到数据，就再使用净值解析逻辑处理一次
        if(EmailTypeConst.VALUATION_EMAIL_TYPE.equals(emailType)){
            long successCount = emailFundNavDTOList.stream().filter(e -> StringUtil.isEmpty(e.getFailReason())).count();
            if(successCount < 1){
                emailParser = emailParserFactory.getInstance(EmailTypeConst.NAV_EMAIL_TYPE);
                emailFundNavDTOList = emailParser.parse(emailContentInfoDTO, emailFieldMap);
                emailFundNavDTOList = emailFundNavDTOList.stream().filter(e -> StringUtil.isEmpty(e.getFailReason())).collect(Collectors.toList());
            }
        }
        return emailFundNavDTOList;
    }

    public void saveRelatedTable(String emailAddress, List<EmailContentInfoDTO> emailContentInfoDTOList, Map<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavMap) {
        String emailTitle = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailTitle() : null;
        String emailDate = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailDate() : null;
        Integer emailType = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailType() : null;
        Integer emailId = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getEmailId() : null;
        String senderEmail = CollUtil.isNotEmpty(emailContentInfoDTOList) ? emailContentInfoDTOList.get(0).getSenderEmail() : null;
        Date parseDate = new Date();
        int emailParseStatus = EmailParseStatusConst.SUCCESS;
        EmailParseInfoDO emailParseInfoDO = buildEmailParseInfo(emailId, emailAddress, senderEmail, emailDate, emailTitle, emailType, emailParseStatus, parseDate);
        emailId = saveEmailParseInfo(emailParseInfoDO);

        // python 报告解析接口结果
        List<ParseResult<ReportData>> dataList = ListUtil.list(false);
        for (Map.Entry<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavEntry : fileNameNavMap.entrySet()) {
            // 保存邮件文件表
            EmailContentInfoDTO emailContentInfoDTO = fileNameNavEntry.getKey();
            String fileName = emailContentInfoDTO.getFileName();
            //针对HTML，只解析净值规模和估值表类型的邮件，其余类型不解析
            if (!Objects.equals(EmailTypeConst.NAV_EMAIL_TYPE, emailType)
                    && !Objects.equals(EmailTypeConst.VALUATION_EMAIL_TYPE, emailType) && fileName.endsWith(".html")) {
                continue;
            }
            Integer fileId = saveEmailFileInfo(emailId, emailContentInfoDTO.getFileId(), fileName, emailContentInfoDTO.getFilePath(), parseDate);
            List<EmailFundNavDTO> fundNavDTOList = fileNameNavEntry.getValue();
            //记录采集表
            if(CollUtil.isNotEmpty(fundNavDTOList)){
                List<EmailFundNavCollectionDO> emailFundNavCollectionVOList = new ArrayList<>();
                Integer finalEmailId = emailId;
                fundNavDTOList.stream().forEach(e -> emailFundNavCollectionVOList.add(toEmailFundNavCollectionVO(e, finalEmailId,fileId,parseDate,emailAddress,senderEmail,emailDate)));
                batchSaveEmailFundNavCollection(emailFundNavCollectionVOList);
            }
            if (CollUtil.isNotEmpty(fundNavDTOList)) {
                // 过滤出解析成功的数据
                fundNavDTOList = fundNavDTOList.stream().filter(e -> e != null && StrUtil.isBlank(e.getFailReason())).toList();
            }
            if (Objects.equals(EmailTypeConst.MONTH_REPORT_EMAIL_TYPE, emailType)
                    ||Objects.equals(EmailTypeConst.YEAR_REPORT_EMAIL_TYPE, emailType)
                    ||Objects.equals(EmailTypeConst.QUARTER_REPORT_EMAIL_TYPE, emailType)) {
                // 解析结果（可以从python获取或者自行解析）并保存报告
                ParseResult<ReportData> parseResult = this.parseReportAndHandleResult(fileId, emailContentInfoDTO);
                dataList.add(parseResult);
            }
            for (EmailFundNavDTO fundNavDTO : fundNavDTOList) {
                // 设置净值数据的解析状态
                setNavParseStatus(fundNavDTO, emailTitle);
            }
            //需要写入到文件管理
            try{
                if (Objects.equals(EmailTypeConst.YEAR_REPORT_EMAIL_TYPE, emailType)
                        ||Objects.equals(EmailTypeConst.QUARTER_REPORT_EMAIL_TYPE, emailType)
                        || Objects.equals(EmailTypeConst.MONTH_REPORT_EMAIL_TYPE, emailType)) {
                    ReportData reportData = dataList.get(0).getData();
                    EmailFundNavDTO emailFundNavDTO = new EmailFundNavDTO();
                    if(reportData != null){
                        ReportFundInfoDTO fundInfoDTO = reportData.getFundInfo();
                        if(fundInfoDTO != null){
                            emailFundNavDTO.setFundName(fundInfoDTO.getFundName());
                            emailFundNavDTO.setRegisterNumber(fundInfoDTO.getRegisterNumber());
                        }
                    }
                    saveFileManage(fileId,emailType,emailContentInfoDTO,ListUtil.toList(emailFundNavDTO),parseDate);
                }else{
                    saveFileManage(fileId,emailType,emailContentInfoDTO,fundNavDTOList,parseDate);
                }
            }catch (Exception e){
                log.error("保存文件管理信息报错："+e.getMessage(),e);
            }
            // 保存净值表和规模表
            saveNavAndAssetNet(fileId, fundNavDTOList, parseDate);
            saveValuationInfo(fileId, fundNavDTOList);
        }

         //更新邮件解析结果 -> 当【净值日期】和【备案编码/基金名称】能正常解读，即识别为【成功】
        long successNavCount = fileNameNavMap.values().stream().flatMap(List::stream).filter(e -> e != null && StrUtil.isBlank(e.getFailReason())).count();
        emailParseStatus = successNavCount >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
        String failReason = null;
        if (emailParseStatus == EmailParseStatusConst.FAIL) {
            // 邮件解析失败时 -> 保存失败原因
            int hasPdfFile = emailContentInfoDTOList.stream().map(EmailContentInfoDTO::getFilePath).anyMatch(ExcelUtil::isPdf) ? 1 : 0;
            List<EmailFundNavDTO> navDTOList = fileNameNavMap.values().stream().flatMap(List::stream).toList();
            //过滤空数据
            navDTOList = navDTOList.stream().filter(e -> (StringUtil.isNotEmpty(e.getNav())||
                    StringUtil.isNotEmpty(e.getCumulativeNavWithdrawal()) ||  StringUtil.isNotEmpty(e.getPriceDate()) ||  StringUtil.isNotEmpty(e.getFundName()) || StringUtil.isNotEmpty(e.getRegisterNumber()))).collect(Collectors.toList());
            failReason = hasPdfFile == 1 && CollUtil.isEmpty(navDTOList) ? NavParseStatusConst.NO_DATA_FROM_PDF_TEXT : navDTOList.stream().map(EmailFundNavDTO::getFailReason).distinct().collect(Collectors.joining("/"));
            if (StrUtil.isBlank(failReason)) {
                failReason = NavParseStatusConst.TEMPLATE_NOT_SUPPORTED_TEXT;
            }
        }
        // 报告邮件有一条失败就表示整个邮件解析失败
        if ((Objects.equals(EmailTypeConst.MONTH_REPORT_EMAIL_TYPE, emailType)
            || Objects.equals(EmailTypeConst.QUARTER_REPORT_EMAIL_TYPE, emailType)
            || Objects.equals(EmailTypeConst.YEAR_REPORT_EMAIL_TYPE, emailType)) && CollUtil.isNotEmpty(dataList)) {
            long sucNum = dataList.stream().filter(e -> Objects.equals(1, e.getStatus())).count();
            if (sucNum > 0) {
                emailParseStatus = EmailParseStatusConst.SUCCESS;
                failReason = null;
            } else {
                emailParseStatus = EmailParseStatusConst.FAIL;
                failReason = dataList.stream().map(ParseResult::getMsg).collect(Collectors.joining("/"));
            }
        }
        emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus, failReason);
    }

    private void saveFileManage(Integer fileId,Integer emailType, EmailContentInfoDTO emailContentInfoDTO,List<EmailFundNavDTO> fundNavDTOList, Date parseDate) {
        try{
            //HTML不保存
            if(ExcelUtil.isHTML(emailContentInfoDTO.getFilePath())){
                return;
            }
            FileManageDO fileManageDO = new FileManageDO();
            fileManageDO.setFileId(Long.valueOf(fileId.intValue()));
            fileManageDO.setFileType(emailType);
            fileManageDO.setSourceType(FileSourceType.SYSTEM_SOURCE);
            fileManageDO.setFileDate(parseDate);
            fileManageDO.setFileName(emailContentInfoDTO.getFileName());
            fileManageDO.setFilePath(emailContentInfoDTO.getFilePath());
            fileManageDO.setCreateTime(DateUtils.getNowDate());
            fileManageDO.setUpdateTime(DateUtils.getNowDate());
            fileManageDO.setIsvalid(1);
            if(CollUtil.isNotEmpty(fundNavDTOList)){
                List<EmailFundNavDTO> fundIdNavList = fundNavDTOList.stream().filter(e -> CollUtil.isNotEmpty(e.getFundIdList())).collect(Collectors.toList());
                if(CollUtil.isNotEmpty(fundIdNavList)){
                    //取第一个有基金ID作为附件关联的基金
                    String fundId = fundIdNavList.get(0).getFundIdList().get(0);
                    fileManageDO.setFundId(fundId);
                    FundInfoDO fundInfoDO = fundInfoMapper.searchFundDetail(fundId);
                    fileManageDO.setRegisterNumber(fundInfoDO.getRegisterNumber());
                    fileManageDO.setFundName(fundInfoDO.getFundName());
                }else{
                    //说明是未匹配基金或者存在数据缺失，取附件上的基金名称和备案编码
                    List<EmailFundNavDTO> fundNameNavList = fundNavDTOList.stream().filter(e -> StringUtil.isNotEmpty(e.getFundName())).collect(Collectors.toList());
                    if(CollUtil.isNotEmpty(fundNameNavList)){
                        EmailFundNavDTO emailFundNavDTO = fundNameNavList.get(0);
                        fileManageDO.setRegisterNumber(emailFundNavDTO.getRegisterNumber());
                        fileManageDO.setFundName(emailFundNavDTO.getFundName());
                    }else{
                        List<EmailFundNavDTO> registerNumbereNavList = fundNavDTOList.stream().filter(e -> StringUtil.isNotEmpty(e.getRegisterNumber())).collect(Collectors.toList());
                        if(CollUtil.isNotEmpty(registerNumbereNavList)){
                            EmailFundNavDTO registerNumbereNav = registerNumbereNavList.get(0);
                            fileManageDO.setRegisterNumber(registerNumbereNav.getRegisterNumber());
                        }
                    }
                }
            }
            FileManageDO oldFileManageDO = fileManageMapper.getFileInfoByFileId(Long.valueOf(fileId.intValue()));
            if(oldFileManageDO == null){
                fileManageMapper.saveFileManage(fileManageDO);
            }else{
                fileManageMapper.updateFileManage(fileManageDO);
            }
        }catch (Exception e){
            log.error(e.getMessage(),e);
        }
    }

    private void batchSaveEmailFundNavCollection(List<EmailFundNavCollectionDO> emailFundNavCollectionVOList) {
        //过滤空数据
        emailFundNavCollectionVOList = emailFundNavCollectionVOList.stream().filter(e -> (StringUtil.isNotEmpty(e.getNav())||
                StringUtil.isNotEmpty(e.getCumulativeNavWithdrawal()) ||  StringUtil.isNotEmpty(e.getPriceDate()) ||  StringUtil.isNotEmpty(e.getFundName()) || StringUtil.isNotEmpty(e.getRegisterNumber()))).collect(Collectors.toList());
        if(CollUtil.isNotEmpty(emailFundNavCollectionVOList)){
            Integer fileId = emailFundNavCollectionVOList.get(0).getFileId();
            Integer emailId = emailFundNavCollectionVOList.get(0).getEmailId();
            emailFundNavCollectionMapper.delete(fileId,emailId);
            try{
                emailFundNavCollectionMapper.batchInsert(emailFundNavCollectionVOList);
            }catch (Exception ex){
                //如果批量插入报错，可能是因为解析的数据超过限制长度，这种情况基本是垃圾数据，直接跳过
                emailFundNavCollectionVOList.forEach(e -> saveEmailFundNavCollection(e));
            }
        }
    }

    private void saveEmailFundNavCollection(EmailFundNavCollectionDO fundNavCollectionDO) {
        try{
            emailFundNavCollectionMapper.batchInsert(CollUtil.toList(fundNavCollectionDO));
        }catch (Exception e){
            //如果是异常，说明是垃圾数据
        }
    }

    private EmailFundNavCollectionDO toEmailFundNavCollectionVO(EmailFundNavDTO fundNavDTO, Integer emailId,Integer fileId,Date parseDate,String email,String senderEmail,String emailDate) {
        EmailFundNavCollectionDO vo = new EmailFundNavCollectionDO();
        vo.setFileId(fileId);
        vo.setEmailId(emailId);
        vo.setFundName(fundNavDTO.getFundName());
        vo.setAssetNet(fundNavDTO.getAssetNet());
        vo.setAssetShare(fundNavDTO.getAssetShare());
        vo.setRegisterNumber(fundNavDTO.getRegisterNumber());
        vo.setPriceDate(fundNavDTO.getPriceDate());
        vo.setNav(fundNavDTO.getNav());
        vo.setExceptionStatus(fundNavDTO.getParseStatus());
        vo.setVirtualNav(fundNavDTO.getVirtualNav());
        vo.setCumulativeNavWithdrawal(fundNavDTO.getCumulativeNavWithdrawal());
        vo.setCreateTime(DateUtils.getNowDate());
        vo.setUpdateTime(DateUtils.getNowDate());
        vo.setIsvalid(1);
        vo.setParseDate(parseDate);
        vo.setEmail(email);
        vo.setSenderEmail(senderEmail);
        vo.setEmailDate(DateUtils.parse(emailDate,DateUtils.YYYY_MM_DD_HH_MM_SS));
        return vo;
    }

    private void saveValuationInfo(Integer fileId, List<EmailFundNavDTO> fundNavDTOList) {
        if (CollUtil.isEmpty(fundNavDTOList)) {
            return;
        }
        valuationTableMapper.unValid(fileId);
        for (EmailFundNavDTO fundNavDTO : fundNavDTOList) {
            ValuationTableDO valuationTableDO = fundNavDTO.getValuationTableDO();
            if (valuationTableDO == null) {
                continue;
            }
            List<String> fundIdList = fundNavDTO.getFundIdList();
            valuationTableDO.setFileId(fileId);

            List<FundPositionDetailDO> fundPositionDetailDOList = fundNavDTO.getFundPositionDetailDOList();
            List<CmValuationTableAttribute> valuationTableAttributeList = fundNavDTO.getValuationTableAttributeList();
            if (CollUtil.isEmpty(fundIdList)) {
                valuationTableMapper.insert(valuationTableDO);
                int valuationId = valuationTableDO.getId();
                saveValuationTableAttribute(valuationId, valuationTableAttributeList);
                fundPositionDetailDOList.forEach(e -> e.setValuationId(valuationId));
                // 不匹配基金的情况下 -> 不写fund_position_detail
                // saveFundPositionDetail(fundPositionDetailDOList, null, fundNavDTO.getPriceDate());
                continue;
            }
            for (String fundId : fundIdList) {
                valuationTableDO.setFundId(fundId);
                valuationTableMapper.insert(valuationTableDO);
                int valuationId = valuationTableDO.getId();
                fundPositionDetailDOList.forEach(e -> {
                    e.setFundId(fundId);
                    e.setValuationId(valuationId);
                });
                saveValuationTableAttribute(valuationId, valuationTableAttributeList);
                saveFundPositionDetail(fundPositionDetailDOList, fundId, fundNavDTO.getPriceDate());
            }
        }
    }

    public void saveFundPositionDetail(List<FundPositionDetailDO> fundPositionDetails, String fundId, String valuationDate) {
        int subBegin = 0;
        int subEnd = stepSize;
        int insertNum = 0;
        int hasStock = 0;
        int hasBond = 0;
        int hasFuture = 0;
        if (CollectionUtil.isNotEmpty(fundPositionDetails)) {
            //插入持仓数据 cm_fund_position_detail（记录数较多，得分批）
            // 先删除原先的数据 然后写入数据
            fundPositionDetailMapper.deleteUnUsed(fundId, valuationDate);
            // 将最终结果写入 cm_fund_position_detail
            while (subBegin < fundPositionDetails.size()) {
                List<FundPositionDetailDO> segment = fundPositionDetails.subList(subBegin, Math.min(subEnd, fundPositionDetails.size()));
                for (FundPositionDetailDO fundPositionDetail : segment) {
                    //实收信托、实收资本对应的编码为107
                    if (StrUtil.isNotBlank(fundPositionDetail.getSecuritiesName()) && "实收信托".equals(fundPositionDetail.getSecuritiesName().trim())) {
                        fundPositionDetail.setSubjectCode("107");
                        fundPositionDetail.setSecuritiesCode("107");
                    }
                    //设置创建者id,在拿取最近修改人时有用
                    fundPositionDetail.setCreatorId(0);
                    if (fundPositionDetail.getLevel() != null && fundPositionDetail.getLevel() >= 4) {
                        Integer secType = fundPositionDetail.getSecType();
                        Integer subType = fundPositionDetail.getSubType();
                        if (secType != null && subType != null) {
                            if (secType == HoldingType.Stock.getId() && subType == HoldingType.Stock.getId()) {
                                hasStock = 1;
                            }
                            // 正逆回购不属于债券
                            String subjectCode = fundPositionDetail.getSubjectCode();
                            String[] SALE_CODES = {"2202", "1202"};
                            if (secType == HoldingType.Bond.getId() && !StrUtil.containsAny(subjectCode, SALE_CODES)) {
                                hasBond = 1;
                            }
                            if (secType == HoldingType.Future.getId() || secType == HoldingType.Option.getId()) {
                                hasFuture = 1;
                            }
                        }
                    }
                }
                insertNum += fundPositionDetailMapper.insertMulti(segment);
                subBegin = subEnd;
                subEnd += stepSize;
            }
        }
        // 更新 cm_user_valuation_table
        if (insertNum > 0) {
            valuationTableMapper.updateUpdateAnalyzed(fundId, valuationDate, hasStock, hasBond, hasFuture);
        }
    }

    private void saveValuationTableAttribute(Integer valuationId, List<CmValuationTableAttribute> valuationTableAttributes) {
        if (valuationId == null || CollUtil.isEmpty(valuationTableAttributes)) {
            return;
        }
        List<ValuationTableAttributeDO> valuationTableAttributeDOList = buildValuationTableAttributeDO(valuationId, valuationTableAttributes);
        valuationTableAttributeMapper.deleteByValuationId(valuationId);
        if (CollUtil.isNotEmpty(valuationTableAttributeDOList)) {
            valuationTableAttributeMapper.batchInsert(valuationTableAttributeDOList);
        }
    }

    private List<ValuationTableAttributeDO> buildValuationTableAttributeDO(Integer valuationId, List<CmValuationTableAttribute> valuationTableAttributes) {
        if (CollUtil.isEmpty(valuationTableAttributes)) {
            return CollUtil.newArrayList();
        }
        return valuationTableAttributes.stream().map(e -> {
            ValuationTableAttributeDO tableAttributeDO = new ValuationTableAttributeDO();
            tableAttributeDO.setValuationId(valuationId);
            tableAttributeDO.setSubjectCode(e.getOriginalSubjectCode());
            tableAttributeDO.setSubjectName(e.getSubjectName());
            tableAttributeDO.setCurrency(e.getCurrency());
            tableAttributeDO.setExchangeRate(e.getExchangeRate());
            tableAttributeDO.setSecuritiesAmount(e.getSecuritiesAmount());
            tableAttributeDO.setUnitCost(e.getUnitCost());
            tableAttributeDO.setOriCurrencyCost(e.getOCurrencyCost());
            tableAttributeDO.setNetCost(e.getNetCost());
            tableAttributeDO.setNetCostRatio(e.getNetCostRatio());
            tableAttributeDO.setMarketPrice(e.getMarketPrice());
            tableAttributeDO.setOriCurrencyMarketValue(e.getOCurrencyMarketValue());
            tableAttributeDO.setMarketValue(e.getMarketValue());
            tableAttributeDO.setMarketValueRatio(e.getMarketValueRatio());
            tableAttributeDO.setOriCurrencyIncrement(e.getOCurrencyMarketValue());
            tableAttributeDO.setIncrement(e.getIncrement());
            tableAttributeDO.setHaltInfo(e.getHaltInfo());
            tableAttributeDO.setRightsInterestsInfo(e.getRightsInterestsInfo());
            tableAttributeDO.setIsvalid(1);
            tableAttributeDO.setCreatorId(0);
            tableAttributeDO.setUpdaterId(0);
            tableAttributeDO.setCreateTime(new Date());
            tableAttributeDO.setUpdateTime(new Date());
            return tableAttributeDO;
        }).collect(Collectors.toList());
    }

    private ParseResult<ReportData> parseReportAndHandleResult(int fileId, EmailContentInfoDTO emailContentInfoDTO) {
        ParseResult<ReportData> result = new ParseResult<>();
        String fileName = emailContentInfoDTO.getFileName();
        Integer emailType = emailContentInfoDTO.getEmailType();
        if (!Objects.equals(EmailTypeConst.MONTH_REPORT_EMAIL_TYPE, emailType)
                ||!Objects.equals(EmailTypeConst.YEAR_REPORT_EMAIL_TYPE, emailType)
                ||!Objects.equals(EmailTypeConst.QUARTER_REPORT_EMAIL_TYPE, emailType)
                || StrUtil.isBlank(fileName)) {
            result.setStatus(ReportParseStatus.NOT_A_REPORT.getCode());
            result.setMsg(ReportParseStatus.NOT_A_REPORT.getMsg());
            return result;
        }
        Pattern pattern = Pattern.compile("S(?:[A-Z]{0}[0-9]{5}|[A-Z][0-9]{4}|[A-Z]{2}[0-9]{3}|[A-Z]{3}[0-9]{2})");
        Matcher matcher = pattern.matcher(fileName);
        String registerNumber = null;
        if (matcher.find()) {
            registerNumber = matcher.group();
        }
        // 类型识别---先识别季度报告，没有季度再识别年度报告，最后识别月报
        ReportType reportType = null;
        if (StrUtil.containsAny(fileName, ReportType.QUARTERLY.getPatterns())) {
            reportType = ReportType.QUARTERLY;
        } else if (StrUtil.containsAny(fileName, ReportType.ANNUALLY.getPatterns())) {
            reportType = ReportType.ANNUALLY;
        } else if (StrUtil.containsAny(fileName, ReportType.MONTHLY.getPatterns())) {
            reportType = ReportType.MONTHLY;
        }
        // 解析器--如果开启python解析则直接调用python接口，否则根据文件后缀获取对应解析器
        ReportParserFileType fileType;
        if (Objects.equals(Boolean.TRUE, this.properties.getEnablePyParser())) {
            fileType = ReportParserFileType.PYTHON;
        } else {
            String fileSuffix = StrUtil.subAfter(fileName, ".", true);
            fileType = ReportParserFileType.getBySuffix(fileSuffix);
        }
        // 不支持的格式
        if (fileType == null) {
            result.setStatus(ReportParseStatus.NO_SUPPORT_TEMPLATE.getCode());
            result.setMsg(StrUtil.format(ReportParseStatus.NO_SUPPORT_TEMPLATE.getMsg(), fileName));
            return result;
        }
        // 不是定期报告的判断逻辑放在不支持的格式下面
        if (reportType == null) {
            result.setStatus(ReportParseStatus.NOT_A_REPORT.getCode());
            result.setMsg(StrUtil.format(ReportParseStatus.NOT_A_REPORT.getMsg(), fileName));
            return result;
        }
        // 解析报告
        ReportData reportData = null;
        StopWatch parserWatch = new StopWatch();
        parserWatch.start();
        try {
            ReportParserParams params = ReportParserParams.builder().fileId(fileId).filename(fileName)
                    .filepath(emailContentInfoDTO.getFilePath()).registerNumber(registerNumber).build();
            ReportParser<ReportData> instance = this.reportParserFactory.getInstance(reportType, fileType);
            reportData = instance.parse(params);
            result.setStatus(1);
            result.setMsg("报告解析成功");
            result.setData(reportData);
        } catch (ReportParseException e) {
            log.error("解析失败\n{}", e.getMsg());
            result.setStatus(e.getCode());
            result.setMsg(e.getMsg());
        } catch (Exception e) {
            log.error("解析错误\n{}", ExceptionUtil.stacktraceToString(e));
            result.setStatus(ReportParseStatus.PARSE_FAIL.getCode());
            result.setMsg(StrUtil.format(ReportParseStatus.PARSE_FAIL.getMsg(), e.getMessage()));
        } finally {
            parserWatch.stop();
            if (log.isInfoEnabled()) {
                log.info("报告{}解析结果为{}，耗时{}ms", fileName, reportData, parserWatch.getTotalTimeMillis());
            }
        }
        // 保存报告解析结果
        if (reportData != null) {
            StopWatch writeWatch = new StopWatch();
            writeWatch.start();
            try {
                ReportWriter<ReportData> instance = this.reportWriterFactory.getInstance(reportType);
                instance.write(reportData);
            } catch (Exception e) {
                log.error("报告{}结果保存失败\n{}", fileName, ExceptionUtil.stacktraceToString(e));
            } finally {
                writeWatch.stop();
                if (log.isInfoEnabled()) {
                    log.info("报告{}解析结果保存完成，耗时{}ms", fileName, writeWatch.getTotalTimeMillis());
                }
            }
        }
        return result;
    }

    private void saveNavAndAssetNet(Integer fileId, List<EmailFundNavDTO> fundNavDTOList, Date parseDate) {
        if (CollUtil.isEmpty(fundNavDTOList)) {
            return;
        }
        // 净值数据
        List<EmailFundNavDO> emailFundNavDOList = fundNavDTOList.stream()
                .map(e -> buildEmailFundNavDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
        if (CollUtil.isNotEmpty(emailFundNavDOList)) {
            // 先删除文件id下的净值数据(考虑到重新解析的需求,如果是首次解析,那么file_id下不存在净值数据)
            emailFundNavMapper.deleteByFileId(fileId);
            emailFundNavMapper.batchInsert(emailFundNavDOList);
            List<NavDO> navDOList = emailFundNavDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
                    .map(e -> BeanUtil.copyProperties(e, NavDO.class)).collect(Collectors.toList());
            saveNavDo(navDOList,fileId);
        }
        // 保存规模数据
        List<EmailFundAssetDO> emailFundAssetDOList = fundNavDTOList.stream()
                .map(e -> buildEmailFundAssetDo(fileId, e, parseDate)).filter(CollUtil::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
        if (CollUtil.isNotEmpty(emailFundAssetDOList)) {
            // 先删除file_id下的规模数据(考虑到重新解析的需求,如果是首次解析,那么file_id下不存在规模数据)
            emailFundAssetMapper.deleteByFileId(fileId);
            emailFundAssetMapper.batchInsert(emailFundAssetDOList);
            List<AssetDO> assetDOList = emailFundAssetDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
                    .map(e -> BeanUtil.copyProperties(e, AssetDO.class)).collect(Collectors.toList());
            saveAssetDo(assetDOList);
        }
    }

    public void saveNavDo(List<NavDO> navDOList,Integer fileId) {
        if (CollUtil.isEmpty(navDOList)) {
            return;
        }
        Map<String, List<NavDO>> fundIdNavMap = navDOList.stream().collect(Collectors.groupingBy(NavDO::getFundId));
        for (Map.Entry<String, List<NavDO>> entry : fundIdNavMap.entrySet()) {
            List<NavDO> navDOS = entry.getValue();
            //按照净值日期排序，方便后面的振幅检测
            if (CollUtil.isNotEmpty(navDOS) && navDOS.size() > 1) {
                navDOS = navDOS.stream().sorted(Comparator.comparing(NavDO::getPriceDate)).collect(Collectors.toList());
            }
            //因为要做振幅检测，所以不能使用批量插入的方式
            for(NavDO insertDO : navDOS){
                //针对净值日期做校验,净值日期不能早于基金成立日,净值日期不能晚于基金清算日期,非成立日净值日期不能是周末
                Date priceDate = insertDO.getPriceDate();
                FundInfoDO fundInfoDO = fundInfoMapper.searchFundDetail(insertDO.getFundId());
                String inceptionDate = fundInfoDO.getInceptionDate();
                if(inceptionDate != null && DateUtils.parse(inceptionDate,DateUtils.YYYY_MM_DD).after(priceDate)){
                    saveEmailFundNav(insertDO,fileId,NavParseStatusConst.PRICE_DATE_BEFORE_INCEPTION,0);
                    continue;
                }
                String liquidateDate = fundInfoDO.getLiquidateDate();
                if(liquidateDate != null && DateUtils.parse(liquidateDate,DateUtils.YYYY_MM_DD).before(priceDate)){
                    saveEmailFundNav(insertDO,fileId,NavParseStatusConst.PRICE_DATE_AFTER_LIQUIDATE,0);
                    continue;
                }
                //周日算每周第一天,非成立日净值日期不能是周末
                Integer dayOfWeek = DateUtil.dayOfWeek(priceDate);
                if((dayOfWeek == 1 || dayOfWeek==7) && !priceDate.equals(inceptionDate)){
                    saveEmailFundNav(insertDO,fileId,NavParseStatusConst.PRICE_DATE_NOT_WORK_DAY,0);
                    continue;
                }
               BigDecimal  amplitude = amplitudeNav(insertDO);
               if(amplitude.compareTo(BigDecimal.valueOf(1)) >= 0){
                   //振幅超过100%不可以入库，要给出提示信息
                   saveEmailFundNav(insertDO,fileId,NavParseStatusConst.AMPLITUDE_ERROR,0);
               }else{
                   //振幅超过20%可以入库，但要给出提示信息
                   if(amplitude.compareTo(BigDecimal.valueOf(0.2)) >= 0){
                       saveEmailFundNav(insertDO,fileId,NavParseStatusConst.AMPLITUDE_EXCEPTION,1);
                   }
                   NavDO oldNavDO = navMapper.queryFundNav(insertDO);
                   if(oldNavDO != null){
                       oldNavDO.setNav(insertDO.getNav());
                       oldNavDO.setCumulativeNavWithdrawal(insertDO.getCumulativeNavWithdrawal());
                       oldNavDO.setUpdateTime(new Date());
                       navMapper.updateNav(oldNavDO);
                   }else{
                       navMapper.saveNav(insertDO);
                   }
               }
            }
        }
    }

    private void saveEmailFundNav(NavDO insertDO,Integer fileId,Integer exceptionStatus,Integer isStored) {
        EmailFundNavDO fundNavDO = new EmailFundNavDO();
        fundNavDO.setFundId(insertDO.getFundId());
        fundNavDO.setNav(insertDO.getNav());
        fundNavDO.setPriceDate(insertDO.getPriceDate());
        fundNavDO.setCumulativeNavWithdrawal(insertDO.getCumulativeNavWithdrawal());
        fundNavDO.setFileId(fileId);
        fundNavDO.setExceptionStatus(exceptionStatus);
        fundNavDO.setIsStored(isStored);
        fundNavDO.setUpdateTime(new Date());
        emailFundNavMapper.update(fundNavDO);
    }

    public void saveAssetDo(List<AssetDO> assetDOList) {
        if (CollUtil.isEmpty(assetDOList)) {
            return;
        }
        Map<String, List<AssetDO>> fundIdNavMap = assetDOList.stream().collect(Collectors.groupingBy(AssetDO::getFundId));
        for (Map.Entry<String, List<AssetDO>> entry : fundIdNavMap.entrySet()) {
            List<AssetDO> assetDOS = entry.getValue();
            List<String> priceDateList = assetDOS.stream().map(AssetDO::getPriceDate).map(e -> DateUtil.format(e, DateConst.YYYY_MM_DD)).collect(Collectors.toList());
            List<String> dateList = assetMapper.queryFundNavByDate(entry.getKey(), priceDateList);
            List<AssetDO> updateAssetDoList = assetDOS.stream().filter(e -> dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
            List<AssetDO> insertAssetDoList = assetDOS.stream().filter(e -> !dateList.contains(DateUtil.format(e.getPriceDate(), DateConst.YYYY_MM_DD))).collect(Collectors.toList());
            if (CollUtil.isNotEmpty(insertAssetDoList)) {
                Map<Date, List<AssetDO>> priceDateAssetDoListMap = insertAssetDoList.stream().collect(Collectors.groupingBy(AssetDO::getPriceDate));
                boolean hasDuplicationDateData = priceDateAssetDoListMap.values().stream().map(List::size).anyMatch(e -> e > 1);
                if (!hasDuplicationDateData) {
                    assetMapper.batchInsert(insertAssetDoList);
                }
                // 要插入的数据中存在相同日期的数据 -> 只能一条一条的插入数据了
                insertAssetDoList.forEach(e -> saveAssetDo(ListUtil.toList(e)));
            }
            if (CollUtil.isNotEmpty(updateAssetDoList)) {
                assetMapper.batchUpdate(updateAssetDoList);
            }
        }
    }

    private List<EmailFundAssetDO> buildEmailFundAssetDo(Integer fileId, EmailFundNavDTO fundNavDTO, Date parseDate) {
        List<EmailFundAssetDO> fundAssetDOList = CollUtil.newArrayList();
        BigDecimal assetNet = StrUtil.isNotBlank(fundNavDTO.getAssetNet()) ? new BigDecimal(fundNavDTO.getAssetNet()) : null;
        BigDecimal assetShare = StrUtil.isNotBlank(fundNavDTO.getAssetShare()) ? new BigDecimal(fundNavDTO.getAssetShare()) : null;
        if (assetNet == null) {
            return fundAssetDOList;
        }
        Integer isStored = fundNavDTO.getParseStatus() != null && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.ASSET_NET_NEGATIVE)
                && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NOT_MATCH)? 1 : 0;
        Date priceDate = DateUtil.parse(fundNavDTO.getPriceDate(), DateConst.YYYY_MM_DD);
        if (CollUtil.isNotEmpty(fundNavDTO.getFundIdList())) {
            for (String fundId : fundNavDTO.getFundIdList()) {
                EmailFundAssetDO emailFundAssetDO = new EmailFundAssetDO();
                emailFundAssetDO.setFileId(fileId);
                emailFundAssetDO.setPriceDate(priceDate);
                emailFundAssetDO.setFundId(fundId);
                emailFundAssetDO.setFundName(fundNavDTO.getFundName());
                emailFundAssetDO.setRegisterNumber(fundNavDTO.getRegisterNumber());
                emailFundAssetDO.setAssetNet(assetNet);
                emailFundAssetDO.setAssetShare(assetShare);
                emailFundAssetDO.setIsStored(isStored);
                emailFundAssetDO.setExceptionStatus(fundNavDTO.getParseStatus());
                emailFundAssetDO.setIsvalid(1);
                emailFundAssetDO.setCreatorId(0);
                emailFundAssetDO.setCreateTime(parseDate);
                emailFundAssetDO.setUpdaterId(0);
                emailFundAssetDO.setUpdateTime(parseDate);
                fundAssetDOList.add(emailFundAssetDO);
            }
        } else {
            EmailFundAssetDO emailFundAssetDO = new EmailFundAssetDO();
            emailFundAssetDO.setFileId(fileId);
            emailFundAssetDO.setPriceDate(priceDate);
            emailFundAssetDO.setFundName(fundNavDTO.getFundName());
            emailFundAssetDO.setRegisterNumber(fundNavDTO.getRegisterNumber());
            emailFundAssetDO.setAssetNet(assetNet);
            emailFundAssetDO.setAssetShare(assetShare);
            emailFundAssetDO.setIsStored(0);//未匹配基金统一不入库
            emailFundAssetDO.setExceptionStatus(fundNavDTO.getParseStatus());
            emailFundAssetDO.setIsvalid(1);
            emailFundAssetDO.setCreatorId(0);
            emailFundAssetDO.setCreateTime(parseDate);
            emailFundAssetDO.setUpdaterId(0);
            emailFundAssetDO.setUpdateTime(parseDate);
            fundAssetDOList.add(emailFundAssetDO);
        }
        return fundAssetDOList;
    }

    private List<EmailFundNavDO> buildEmailFundNavDo(Integer fileId, EmailFundNavDTO fundNavDTO, Date parseDate) {
        List<EmailFundNavDO> fundNavDOList = CollUtil.newArrayList();
        Date priceDate = DateUtil.parse(fundNavDTO.getPriceDate(), DateConst.YYYY_MM_DD);
        BigDecimal nav = StrUtil.isNotBlank(fundNavDTO.getNav()) ? new BigDecimal(fundNavDTO.getNav()) : null;
        BigDecimal cumulativeNavWithdrawal = StrUtil.isNotBlank(fundNavDTO.getCumulativeNavWithdrawal()) ? new BigDecimal(fundNavDTO.getCumulativeNavWithdrawal()) : null;
        Integer isStored = fundNavDTO.getParseStatus() != null && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NAV_DEFICIENCY)
                && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NOT_MATCH) && !fundNavDTO.getParseStatus().equals(NavParseStatusConst.NAV_NEGATIVE) ? 1 : 0;
        if (CollUtil.isNotEmpty(fundNavDTO.getFundIdList())) {
            for (String fundId : fundNavDTO.getFundIdList()) {
                EmailFundNavDO emailFundNavDO = new EmailFundNavDO();
                emailFundNavDO.setFileId(fileId);
                emailFundNavDO.setIsStored(isStored);
                emailFundNavDO.setPriceDate(priceDate);
                emailFundNavDO.setNav(nav);
                emailFundNavDO.setFundId(fundId);
                emailFundNavDO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
                emailFundNavDO.setFundName(fundNavDTO.getFundName());
                emailFundNavDO.setRegisterNumber(fundNavDTO.getRegisterNumber());
                emailFundNavDO.setExceptionStatus(fundNavDTO.getParseStatus());
                emailFundNavDO.setTemplateId(fundNavDTO.getTemplateId());
                emailFundNavDO.setIsvalid(1);
                emailFundNavDO.setCreatorId(0);
                emailFundNavDO.setCreateTime(parseDate);
                emailFundNavDO.setUpdaterId(0);
                emailFundNavDO.setUpdateTime(parseDate);
                fundNavDOList.add(emailFundNavDO);
            }
        } else {
            EmailFundNavDO emailFundNavDO = new EmailFundNavDO();
            emailFundNavDO.setFileId(fileId);
            emailFundNavDO.setPriceDate(priceDate);
            emailFundNavDO.setNav(nav);
            emailFundNavDO.setCumulativeNavWithdrawal(cumulativeNavWithdrawal);
            emailFundNavDO.setFundName(fundNavDTO.getFundName());
            emailFundNavDO.setRegisterNumber(fundNavDTO.getRegisterNumber());
            emailFundNavDO.setExceptionStatus(fundNavDTO.getParseStatus());
            emailFundNavDO.setTemplateId(fundNavDTO.getTemplateId());
            emailFundNavDO.setIsStored(0);//无法识别基金ID的统一不入库
            emailFundNavDO.setIsvalid(1);
            emailFundNavDO.setCreatorId(0);
            emailFundNavDO.setCreateTime(parseDate);
            emailFundNavDO.setUpdaterId(0);
            emailFundNavDO.setUpdateTime(parseDate);
            fundNavDOList.add(emailFundNavDO);
        }
        return fundNavDOList;
    }

    private Integer saveEmailFileInfo(Integer emailId, Integer fileId, String fileName, String filePath, Date parseDate) {
        EmailFileInfoDO emailFileInfoDO = buildEmailFileInfoDO(emailId, fileId, fileName, filePath, parseDate);
        if (emailFileInfoDO.getId() != null) {
            emailFileInfoMapper.updateTimeById(fileId, parseDate);
            return emailFileInfoDO.getId();
        }
        emailFileInfoMapper.insert(emailFileInfoDO);
        return emailFileInfoDO.getId();
    }

    private EmailFileInfoDO buildEmailFileInfoDO(Integer emailId, Integer fileId, String fileName, String filePath, Date parseDate) {
        EmailFileInfoDO emailFileInfoDO = new EmailFileInfoDO();
        emailFileInfoDO.setId(fileId);
        emailFileInfoDO.setEmailId(emailId);
        emailFileInfoDO.setFileName(fileName);
        emailFileInfoDO.setFilePath(filePath);
        emailFileInfoDO.setIsvalid(1);
        emailFileInfoDO.setCreatorId(0);
        emailFileInfoDO.setCreateTime(parseDate);
        emailFileInfoDO.setUpdaterId(0);
        emailFileInfoDO.setUpdateTime(parseDate);
        return emailFileInfoDO;
    }

    private void setNavParseStatus(EmailFundNavDTO fundNavDTO, String emailTitle) {
        // 1.单位净值或累计净值缺失
        if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal())) {
            fundNavDTO.setParseStatus(NavParseStatusConst.NAV_DEFICIENCY);
            return;
        }
        // 考虑单独规模文件时 -> 无单位净值和累计净值
        // 2.单位净值或累计净值不大于0
        if (!emailTitle.contains("规模")) {
            if (StrUtil.isBlank(fundNavDTO.getNav()) || StrUtil.isBlank(fundNavDTO.getCumulativeNavWithdrawal())
                    || (fundNavDTO.getNav().compareTo("0") <= 0 || fundNavDTO.getCumulativeNavWithdrawal().compareTo("0") <= 0)) {
                fundNavDTO.setParseStatus(NavParseStatusConst.NAV_NEGATIVE);
                return;
            }
        }
        // 3.资产净值不大于0
        if (StrUtil.isNotBlank(fundNavDTO.getAssetNet()) && fundNavDTO.getAssetNet().compareTo("0") <= 0) {
            fundNavDTO.setParseStatus(NavParseStatusConst.ASSET_NET_NEGATIVE);
            return;
        }
        //净值日期不能大于当前日期
        if(StringUtil.isNotEmpty(fundNavDTO.getPriceDate())){
            if(DateUtils.parse(fundNavDTO.getPriceDate(),DateUtils.YYYY_MM_DD).after(new Date())){
                fundNavDTO.setParseStatus(NavParseStatusConst.PRICE_DATE_AFTER_NOW);
                return;
            }
        }
        // 4.匹配基金(考虑到解析估值表时已经匹配上基金的情况)
        List<String> fundIdList = fundNavDTO.getFundIdList();
        if (CollUtil.isEmpty(fundIdList)) {
            fundIdList = fundService.getFundIdByNamesAndCode(fundNavDTO.getFundName(), fundNavDTO.getRegisterNumber());
            if (CollUtil.isEmpty(fundIdList)) {
                fundNavDTO.setParseStatus(NavParseStatusConst.NOT_MATCH);
            }
        }
        fundNavDTO.setFundIdList(fundIdList);
        // 写入别名管理表fund_alias
        saveFundAlias(fundNavDTO.getFundName(), fundNavDTO.getRegisterNumber(), fundIdList);
        if (CollUtil.isEmpty(fundIdList)) {
            return;
        }
        fundNavDTO.setParseStatus(NavParseStatusConst.SUCCESS);
    }
    //振幅检测
    private BigDecimal amplitudeNav(NavDO navDO) {
        String fundId = navDO.getFundId();
        String priceDate = DateUtils.format(navDO.getPriceDate(), DateUtils.YYYY_MM_DD);
        //上一期单位净值
        NavDO preNavDO = navMapper.queryLastNavByFundIdDate(fundId,priceDate);
        if(preNavDO == null){
            return BigDecimal.valueOf(0);
        }
        //|（当期-上期）/ 上期 |
        BigDecimal amplitudeRate = navDO.getNav().subtract(preNavDO.getNav()).divide(preNavDO.getNav(),4, BigDecimal.ROUND_HALF_UP);
        return amplitudeRate.abs();
    }

    private void saveFundAlias(String fundName, String registerNumber, List<String> fundIdList) {
        // 未识别到基金名称和备案编码的数据不写入别名管理
        if (StrUtil.isBlank(fundName) && StrUtil.isBlank(registerNumber)) {
            return;
        }
        List<FundAliasDO> fundAliasDOList = CollUtil.newArrayList();
        if (StrUtil.isNotBlank(fundName) && StrUtil.isNotBlank(registerNumber)) {
            fundAliasDOList = fundAliasMapper.queryFundIdByNameAndRegisterNumber(fundName, registerNumber);
        }
        if (StrUtil.isBlank(fundName) && StrUtil.isNotBlank(registerNumber)) {
            fundAliasDOList = fundAliasMapper.queryFundIdByRegisterNumber(registerNumber);
        }
        if (StrUtil.isNotBlank(fundName) && StrUtil.isBlank(registerNumber)) {
            fundAliasDOList = fundAliasMapper.queryFundIdByName(fundName);
        }

        // 未匹配基金且已写入别名表
        if (CollUtil.isEmpty(fundIdList) && CollUtil.isNotEmpty(fundAliasDOList)) {
            return;
        }
        // 未匹配基金且未写入别名表
        if (CollUtil.isEmpty(fundIdList) && CollUtil.isEmpty(fundAliasDOList)) {
            fundAliasMapper.batchInsert(ListUtil.toList(buildFundAliasDO(fundName, registerNumber, null)));
            log.info("写入别名表 -> 基金名称:{},备案编码:{},基金id:{}", fundName, registerNumber, fundIdList);
            return;
        }
        // 匹配上基金 -> 需要判断此时基金id是否已经在别名管理表
        if (CollUtil.isNotEmpty(fundAliasDOList)) {
            List<String> collect = fundAliasDOList.stream().filter(Objects::nonNull).map(FundAliasDO::getTargetFundId).filter(Objects::nonNull).distinct().toList();
            fundIdList = fundIdList.stream().filter(e -> !collect.contains(e)).toList();
        }
        List<FundAliasDO> fundAliasDOS = CollUtil.isNotEmpty(fundIdList) ? fundIdList.stream().map(e -> buildFundAliasDO(fundName, registerNumber, e)).toList() : null;
        if (CollUtil.isNotEmpty(fundAliasDOS)) {
            fundAliasMapper.batchInsert(fundAliasDOS);
            log.info("写入别名表 -> 基金名称:{},备案编码:{},基金id:{}", fundName, registerNumber, fundIdList);
        }
    }

    public FundAliasDO buildFundAliasDO(String fundName, String registerNumber, String fundId) {
        FundAliasDO fundAliasDO = new FundAliasDO();
        fundAliasDO.setTargetFundId(fundId);
        fundAliasDO.setSourceFundName(fundName);
        fundAliasDO.setSourceRegisterNumber(registerNumber);
        fundAliasDO.setIsvalid(1);
        fundAliasDO.setCreatorId(0);
        fundAliasDO.setCreateTime(new Date());
        fundAliasDO.setUpdateTime(new Date());
        fundAliasDO.setUpdaterId(0);
        return fundAliasDO;
    }

    private Integer saveEmailParseInfo(EmailParseInfoDO emailParseInfoDO) {
        if (emailParseInfoDO == null) {
            return null;
        }
        // 重新邮件功能 -> 修改解析时间和更新时间
        if (emailParseInfoDO.getId() != null) {
            emailParseInfoMapper.updateParseTime(emailParseInfoDO.getId(), emailParseInfoDO.getParseDate());
            return emailParseInfoDO.getId();
        }
        emailParseInfoMapper.insert(emailParseInfoDO);
        return emailParseInfoDO.getId();
    }

    private EmailParseInfoDO buildEmailParseInfo(Integer emailId, String emailAddress, String senderEmail, String emailDate,
                                                 String emailTitle, Integer emailType, Integer parseStatus, Date parseDate) {
        EmailParseInfoDO emailParseInfoDO = new EmailParseInfoDO();
        emailParseInfoDO.setId(emailId);
        emailParseInfoDO.setSenderEmail(senderEmail);
        emailParseInfoDO.setEmail(emailAddress);
        emailParseInfoDO.setEmailDate(DateUtil.parse(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS));
        emailParseInfoDO.setParseDate(parseDate);
        emailParseInfoDO.setEmailTitle(emailTitle);
        emailParseInfoDO.setEmailType(emailType);
        emailParseInfoDO.setParseStatus(parseStatus);
        emailParseInfoDO.setIsvalid(1);
        emailParseInfoDO.setCreatorId(0);
        emailParseInfoDO.setCreateTime(parseDate);
        emailParseInfoDO.setUpdaterId(0);
        emailParseInfoDO.setUpdateTime(parseDate);
        return emailParseInfoDO;
    }

    public Map<String, List<String>> getEmailFieldMapping() {
        List<EmailFieldMappingDO> emailFieldMappingDOList = emailFieldMapper.getEmailFieldMapping(1);
        return emailFieldMappingDOList.stream()
                .collect(Collectors.toMap(EmailFieldMappingDO::getCode, v -> Arrays.stream(v.getName().split(",")).toList()));
    }

    public Map<Integer, List<String>> getEmailType() {
        Map<Integer, List<String>> emailTypeMap = MapUtil.newHashMap(3, true);
        EmailTypeRuleDO emailTypeRuleDO = emailTypeRuleMapper.getEmailTypeRule();
        String nav = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getNav()) ? emailTypeRuleDO.getNav() : "";
        String valuation = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getValuation()) ? emailTypeRuleDO.getValuation() : "";
        String monthReport = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getMonthReport()) ? emailTypeRuleDO.getMonthReport() : "";
        String quarterReport = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getQuarterReport()) ? emailTypeRuleDO.getQuarterReport() : "";
        String yearReport = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getYearReport()) ? emailTypeRuleDO.getYearReport() : "";
        String changeReport = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getChangeReport()) ? emailTypeRuleDO.getChangeReport() : "";
        String distribution = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getDistribution()) ? emailTypeRuleDO.getDistribution() : "";
        String contract = emailTypeRuleDO != null && StrUtil.isNotBlank(emailTypeRuleDO.getContract()) ? emailTypeRuleDO.getContract() : "";
        emailTypeMap.put(EmailTypeConst.VALUATION_EMAIL_TYPE, Arrays.stream(valuation.split(",")).toList());
        emailTypeMap.put(EmailTypeConst.MONTH_REPORT_EMAIL_TYPE, Arrays.stream(monthReport.split(",")).toList());
        emailTypeMap.put(EmailTypeConst.QUARTER_REPORT_EMAIL_TYPE, Arrays.stream(quarterReport.split(",")).toList());
        emailTypeMap.put(EmailTypeConst.YEAR_REPORT_EMAIL_TYPE, Arrays.stream(yearReport.split(",")).toList());
        emailTypeMap.put(EmailTypeConst.CHANGE_EMAIL_TYPE, Arrays.stream(changeReport.split(",")).toList());
        emailTypeMap.put(EmailTypeConst.CONTRACT_EMAIL_TYPE, Arrays.stream(contract.split(",")).toList());
        emailTypeMap.put(EmailTypeConst.DISTRIBUTION_EMAIL_TYPE, Arrays.stream(distribution.split(",")).toList());
        emailTypeMap.put(EmailTypeConst.NAV_EMAIL_TYPE, Arrays.stream(nav.split(",")).toList());
        return emailTypeMap;
    }

    /**
     * 读取邮件
     *
     * @param mailboxInfoDTO 邮箱配置信息
     * @param emailTypeMap   邮件类型识别规则映射表
     * @param startDate      邮件起始日期
     * @param endDate        邮件截止日期(为null,将解析邮件日期小于等于startDate的当天邮件)
     * @return 读取到的邮件信息
     * @throws Exception 异常信息
     */
    private Map<String, List<EmailContentInfoDTO>> realEmail(MailboxInfoDTO mailboxInfoDTO, Map<Integer, List<String>> emailTypeMap, Date startDate, Date endDate) throws Exception {
        Store store = EmailUtil.getStoreNew(mailboxInfoDTO);
        if (store == null) {
            return MapUtil.newHashMap();
        }
        // 默认读取收件箱的邮件
        Folder folder = store.getFolder("INBOX");
        folder.open(Folder.READ_ONLY);
        Message[] messages = getEmailMessage(folder, mailboxInfoDTO.getProtocol(), startDate);
        if (messages == null || messages.length == 0) {
            log.info("获取不到邮件 -> 邮箱信息:{},开始时间:{},结束时间:{}", mailboxInfoDTO, startDate, endDate);
            return MapUtil.newHashMap();
        }
        Map<String, List<EmailContentInfoDTO>> emailMessageMap = MapUtil.newHashMap();
        for (Message message1 : messages) {
            MimeMessage message = (MimeMessage) message1;
            List<EmailContentInfoDTO> emailContentInfoDTOList = CollUtil.newArrayList();
            String uuidKey = UUID.randomUUID().toString().replaceAll("-", "");
            Integer emailType;
            String senderEmail;
            try {
                Date emailDate = message.getSentDate();
                boolean isNotParseConditionSatisfied = emailDate == null || (endDate != null && emailDate.compareTo(endDate) > 0) || (startDate != null && emailDate.compareTo(startDate) < 0);
                if (isNotParseConditionSatisfied) {
                    continue;
                }
                senderEmail = getSenderEmail(message);
                emailType = EmailUtil.getEmailTypeBySubject(message.getSubject(), emailTypeMap);
                String emailDateStr = DateUtil.format(emailDate, DateConst.YYYY_MM_DD_HH_MM_SS);
                if (emailType == null) {
                    log.info("邮件不满足解析条件 -> 邮件主题:{},邮件日期:{}", message.getSubject(), emailDateStr);
                    continue;
                }
                log.info("邮件采集成功 -> 邮件主题:{},邮件日期:{}", message.getSubject(), emailDateStr);
                Object content = null;
                try{
                    content = message.getContent();
                }catch (Exception e){
                    log.error(e.getMessage(),e);
                    message = new MimeMessage(message);
                    content = message.getContent();
                }
                // 1.邮件为MIME多部分消息体:可能既有邮件又有正文
                if (content instanceof MimeMultipart) {
                    emailContentInfoDTOList = EmailUtil.collectMimeMultipart(message, mailboxInfoDTO.getAccount(), path);
                }
                // 2.邮件只有正文
                if (content instanceof String) {
                    EmailContentInfoDTO emailContentInfoDTO = new EmailContentInfoDTO();
                    emailContentInfoDTO.setEmailContent(content.toString());
                    emailContentInfoDTO.setEmailDate(emailDateStr);
                    emailContentInfoDTO.setEmailTitle(message.getSubject());
                    String fileName = message.getSubject() + DateUtil.format(emailDate, DateConst.YYYYMMDDHHMMSS24) + ".html";
                    fileName = fileName.replaceAll(" ","");
                    String filePath = path + mailboxInfoDTO.getAccount() + "/" + DateUtil.format(emailDate, DateConst.YYYY_MM_DD) + "/" + fileName;
                    File saveFile = new File(filePath);
                    saveFile.setReadable(true);
                    if (!saveFile.exists()) {
                        if (!saveFile.getParentFile().exists()) {
                            saveFile.getParentFile().mkdirs();
                            saveFile.getParentFile().setExecutable(true);
                        }
                    }
                    FileUtil.writeFile(filePath, content.toString());
                    emailContentInfoDTO.setFilePath(filePath);
                    emailContentInfoDTO.setFileName(fileName);
                    emailContentInfoDTOList.add(emailContentInfoDTO);
                }
                if (CollUtil.isNotEmpty(emailContentInfoDTOList)) {
                    //除了净值规模，其余邮件类型不展示正文html文件
                    if (!emailType.equals(EmailTypeConst.NAV_EMAIL_TYPE)) {
                        emailContentInfoDTOList = emailContentInfoDTOList.stream().filter(e -> !ExcelUtil.isHTML(e.getFilePath())).toList();
                    }
                    emailContentInfoDTOList.forEach(e -> {
                        e.setEmailType(emailType);
                        e.setSenderEmail(senderEmail);
                    });
                    emailMessageMap.put(uuidKey, emailContentInfoDTOList);
                }
            } catch (Exception e) {
                log.error("获取邮箱的邮件报错,堆栈信息:{}", ExceptionUtil.stacktraceToString(e));
            }
        }
        folder.close(false);
        store.close();
        return emailMessageMap;
    }

    private String getSenderEmail(MimeMessage message) {
        Address[] senderAddress = null;
        try {
            senderAddress = message.getFrom();
            if (senderAddress == null || senderAddress.length == 0) {
                log.info("发件人获取失败=============================");
                return null;
            }
            // 此时的address是含有编码(MIME编码方式)后的文本和实际的邮件地址
            String address = "";
            for(Address from : senderAddress){
                if(StringUtil.isNotEmpty(from.toString())){
                    address = from.toString();
                    break;
                }
            }
            log.info("发件人地址："+address+"========================senderAddress size:"+senderAddress.length);
            // 正则表达式匹配邮件地址
            Pattern pattern = Pattern.compile("<(\\S+)>");
            Matcher matcher = pattern.matcher(address);
            if (matcher.find()) {
                return matcher.group(1);
            }
            //说明匹配不到，直接获取sender
            Address sender = message.getSender();
            if(sender == null){
                return address;
            }
            String senderEmail = sender.toString();
            log.info("senderEmail:"+senderEmail+"====================");
            if(senderEmail.contains("<") && senderEmail.contains(">") && senderEmail.indexOf("<") < senderEmail.indexOf(">")){
                senderEmail = senderEmail.substring(senderEmail.indexOf("<")+1,senderEmail.length()-1);
            }
            return senderEmail;
        } catch (MessagingException e) {
            log.error(e.getMessage(),e);
        }
        return null;
    }

    public List<NameValueVO> searchEmailCount(DataboardQuery databoardQuery) {
        if(StringUtil.isNotEmpty(databoardQuery.getEndDate())){
            databoardQuery.setEndDate(com.smppw.utils.DateUtil.getAroundDate(DateUtils.parse(databoardQuery.getEndDate(),DateUtils.YYYY_MM_DD),1));
        }
        List<Map<String, Object>> dataList = emailParseInfoMapper.searchEmailDataBoard(databoardQuery);
        List<NameValueVO> result = new ArrayList<>();
        Long total = 0L;
        for (Map<String, Object> data : dataList) {
            NameValueVO vo = new NameValueVO();
            if (1 == ((Integer) data.get("parse_status")).intValue()) {
                vo.setValue((Long) data.get("total"));
                vo.setName("解析成功");
            } else {
                vo.setValue((Long) data.get("total"));
                vo.setName("解析失败");
            }
            total += (Long) data.get("total");
            result.add(vo);
        }
        NameValueVO vo = new NameValueVO();
        vo.setName("解析总数");
        vo.setValue(total);
        result.add(vo);
        return result;
    }

    public List<NameValueVO> searchEmailTypeCount(DataboardQuery databoardQuery) {
        if(StringUtil.isNotEmpty(databoardQuery.getEndDate())){
            databoardQuery.setEndDate(com.smppw.utils.DateUtil.getAroundDate(DateUtils.parse(databoardQuery.getEndDate(),DateUtils.YYYY_MM_DD),1));
        }
        //邮件类型，1-净值，2-估值表，3-定期报告
        List<Map<String, Object>> dataList = emailParseInfoMapper.searchEmailTypeCount(databoardQuery);
        List<NameValueVO> result = new ArrayList<>();
        Integer total = 0;
        for (Map<String, Object> data : dataList) {
            NameValueVO vo = new NameValueVO();
            Integer emailType = (Integer) data.get("email_type");
            Long totalType = (Long) data.get("total");
            if (1 == emailType) {
                vo.setName("净值规模");
                vo.setValue(totalType);
            } else if (2 == emailType) {
                vo.setName("估值表");
                vo.setValue(totalType);
            } else {
                vo.setName("定期报告");
                vo.setValue(totalType);
            }
            result.add(vo);
        }
        return result;
    }

    public EmailParseFailAnalysisVO parseFailAnalysis(DataboardQuery databoardQuery) {
        if(StringUtil.isNotEmpty(databoardQuery.getEndDate())){
            databoardQuery.setEndDate(com.smppw.utils.DateUtil.getAroundDate(DateUtils.parse(databoardQuery.getEndDate(),DateUtils.YYYY_MM_DD),1));
        }
        EmailParseFailAnalysisVO result = new EmailParseFailAnalysisVO();
        //净值规模
        //净值规模
        List<NameValueVO> navNameValueVOS = new ArrayList<>();
        NameValueVO pdfNoDataVO = new NameValueVO();
        databoardQuery.setEmailType(1);
        Long pdfNoData = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, NavParseStatusConst.NO_DATA_FROM_PDF_TEXT);
        pdfNoDataVO.setName(NavParseStatusConst.NO_DATA_FROM_PDF_TEXT);
        pdfNoDataVO.setValue(pdfNoData);
        navNameValueVOS.add(pdfNoDataVO);
        NameValueVO priceDateMissVO = new NameValueVO();
        Long priceDateMiss = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, NavParseStatusConst.PRICE_DATE_IS_NULL_TEXT);
        priceDateMissVO.setValue(priceDateMiss);
        priceDateMissVO.setName(NavParseStatusConst.PRICE_DATE_IS_NULL_TEXT);
        navNameValueVOS.add(priceDateMissVO);
        NameValueVO priceDateErrorVO = new NameValueVO();
        Long priceDateError = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, NavParseStatusConst.PRICE_DATE_IS_ERROR_TEXT);
        priceDateErrorVO.setValue(priceDateError);
        priceDateErrorVO.setName(NavParseStatusConst.PRICE_DATE_IS_ERROR_TEXT);
        navNameValueVOS.add(priceDateErrorVO);
        NameValueVO navMissVO = new NameValueVO();
        Long navMiss = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, NavParseStatusConst.NAV_ASSET_IS_NULL_TEXT);
        navMissVO.setName(NavParseStatusConst.NAV_ASSET_IS_NULL_TEXT);
        navMissVO.setValue(navMiss);
        navNameValueVOS.add(navMissVO);
        NameValueVO navAssetErrorVO = new NameValueVO();
        Long navAssetError = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, NavParseStatusConst.NAV_ASSET_IS_ERROR_TEXT);
        navAssetErrorVO.setName(NavParseStatusConst.NAV_ASSET_IS_ERROR_TEXT);
        navAssetErrorVO.setValue(navAssetError);
        navNameValueVOS.add(navAssetErrorVO);
        NameValueVO fundNameNumberMissVO = new NameValueVO();
        Long fundNameNumberMiss = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, NavParseStatusConst.FUND_NAME_REGISTER_IS_NULL_TEXT);
        fundNameNumberMissVO.setName(NavParseStatusConst.FUND_NAME_REGISTER_IS_NULL_TEXT);
        fundNameNumberMissVO.setValue(fundNameNumberMiss);
        navNameValueVOS.add(fundNameNumberMissVO);
        result.setNavFailAnalysisVO(navNameValueVOS);
        NameValueVO templateUnSupportVO = new NameValueVO();
        Long templateUnSupport = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, NavParseStatusConst.TEMPLATE_NOT_SUPPORTED_TEXT);
        templateUnSupportVO.setName(NavParseStatusConst.TEMPLATE_NOT_SUPPORTED_TEXT);
        templateUnSupportVO.setValue(templateUnSupport);
        navNameValueVOS.add(templateUnSupportVO);
        result.setNavFailAnalysisVO(navNameValueVOS);

        //估值表规模
        databoardQuery.setEmailType(2);
        List<NameValueVO> valuationNameValueVOS = new ArrayList<>();
        NameValueVO fileTypeErrorVO = new NameValueVO();
        Long fileTypeError = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, "文件格式错误");
        fileTypeErrorVO.setName("文件格式错误");
        fileTypeErrorVO.setValue(fileTypeError);
        valuationNameValueVOS.add(fileTypeErrorVO);
        NameValueVO columnMissVO = new NameValueVO();
        Long columnMiss = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, "无市值列或无数量列");
        columnMissVO.setName("无市值列或无数量列");
        columnMissVO.setValue(columnMiss);
        valuationNameValueVOS.add(columnMissVO);
        NameValueVO numbericMissVO = new NameValueVO();
        Long numbericMiss = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, "非数值数据");
        numbericMissVO.setName("非数值数据");
        numbericMissVO.setValue(numbericMiss);
        valuationNameValueVOS.add(numbericMissVO);
        NameValueVO noDataVO = new NameValueVO();
        Long noData = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, "无数据");
        noDataVO.setValue(noData);
        noDataVO.setName("无数据");
        valuationNameValueVOS.add(noDataVO);
        NameValueVO templateErrorVO = new NameValueVO();
        Long templateError = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, "模板不支持");
        templateErrorVO.setValue(templateError);
        templateErrorVO.setName("模板不支持");
        valuationNameValueVOS.add(templateErrorVO);
        result.setValuationFailAnalysisVO(valuationNameValueVOS);
        //定期报告
        /**
         *  PARSE_FAIL(21000, "定期报告解析错误：{}"),
         *     NOT_A_REPORT(21001, "[{}]不是定期报告"),
         *     REPORT_IS_SCAN(21002, "报告[{}]为扫描件"),
         *     NO_SUPPORT_TEMPLATE(21003, "报告[{}]是不支持的文件格式"),
         *     NOT_A_FIXED_FORMAT(21004, "报告[{}]不是基协统一格式"),
         *
         *     PARSE_FUND_INFO_FAIL(21010, "报告[{}]没有解析到基金基本信息"),
         *     PARSE_NAV_INFO_FAIL(21011, "报告[{}]没有解析到基金净值信息"),
         *     PARSE_FINANCIAL_INFO_FAIL(21012, "报告[{}]没有解析到基金财务指标信息"),
         *     PARSE_INDUSTRY_INFO_FAIL(21013, "报告[{}]没有解析到基金行业配置信息"),
         *     PARSE_ASSET_INFO_FAIL(21014, "报告[{}]没有解析到基金资产配置信息"),
         *     PARSE_SHARE_INFO_FAIL(21015, "报告[{}]没有解析到基金份额变动信息"),
         */
        databoardQuery.setEmailType(3);
        List<NameValueVO> reportNameValueVOS = new ArrayList<>();
        NameValueVO scannedFileVO = new NameValueVO();
        Long scannedFile = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, "为扫描件");
        scannedFileVO.setName("报告为扫描件");
        scannedFileVO.setValue(scannedFile);
        reportNameValueVOS.add(scannedFileVO);
        NameValueVO errorAmacFileTypeVO = new NameValueVO();
        Long errorAmacFileType = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, "不是基协统一格式");
        errorAmacFileTypeVO.setName("报告不是基协统一格式");
        errorAmacFileTypeVO.setValue(errorAmacFileType);
        reportNameValueVOS.add(errorAmacFileTypeVO);
        NameValueVO watermarkFileErrorVO = new NameValueVO();
        Long watermarkFileError = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, "没有解析到");
        watermarkFileErrorVO.setName("报告水印干扰导致部分没有解析");
        watermarkFileErrorVO.setValue(watermarkFileError);
        reportNameValueVOS.add(watermarkFileErrorVO);
        NameValueVO noReportVO = new NameValueVO();
        Long noReport = emailParseInfoMapper.countNoDataByFailReason(databoardQuery, "不是定期报告");
        noReportVO.setName("报告不是定期报告");
        noReportVO.setValue(noReport);
        reportNameValueVOS.add(noReportVO);
        result.setReportFailAnalysisVO(reportNameValueVOS);
        return result;
    }

    public EmailParseDataViewVO dataOverview(DataboardQuery databoardQuery) {
        EmailParseDataViewVO dataViewVO = new EmailParseDataViewVO();
        dataViewVO.setEmailNum(emailParseInfoMapper.countEmailTotal(null));
        dataViewVO.setNavEmailNum(emailParseInfoMapper.countEmailTotal(1));
        dataViewVO.setValuationEmailNum(emailParseInfoMapper.countEmailTotal(2));
        dataViewVO.setReportEmailNum(emailParseInfoMapper.countEmailTotal(3));
        dataViewVO.setParseNavNum(emailFundNavMapper.countEmailNavTotal());
        dataViewVO.setParseAssetNum(emailFundAssetMapper.countEmailAssetTotal());
        dataViewVO.setNavNum(navMapper.countNavTotal());
        dataViewVO.setAssetNum(assetMapper.countAssetTotal());
        dataViewVO.setDistribute(distributionMapper.countDistributionTotal());
        dataViewVO.setFundNum(fundInfoMapper.countFundTotal());
        dataViewVO.setCompanyNum(companyInformationMapper.countCompanyTotal());
        return dataViewVO;
    }

    private Message[] getEmailMessage(Folder folder, String protocol, Date startDate) {
        try {
            if (protocol.contains("imap")) {
                // 获取邮件日期大于等于startDate的邮件(搜索条件只支持按天)
                SearchTerm startDateTerm = new ReceivedDateTerm(ComparisonTerm.GE, startDate);
                return folder.search(startDateTerm);
            } else {
                return folder.getMessages();
            }
        } catch (MessagingException e) {
            throw new RuntimeException(e);
        }
    }

    private static class PythonData {
        private Integer fileId;
        private Integer status;
        private String msg;
        private String register_number;

        public Integer getFileId() {
            return fileId;
        }

        public void setFileId(Integer fileId) {
            this.fileId = fileId;
        }

        public Integer getStatus() {
            return status;
        }

        public void setStatus(Integer status) {
            this.status = status;
        }

        public String getMsg() {
            return msg;
        }

        public void setMsg(String msg) {
            this.msg = msg;
        }

        public String getRegister_number() {
            return register_number;
        }

        public void setRegister_number(String register_number) {
            this.register_number = register_number;
        }
    }
}