|
@@ -11,6 +11,7 @@ import cn.hutool.core.util.StrUtil;
|
|
import com.simuwang.base.common.conts.*;
|
|
import com.simuwang.base.common.conts.*;
|
|
import com.simuwang.base.common.enums.ReportParserFileType;
|
|
import com.simuwang.base.common.enums.ReportParserFileType;
|
|
import com.simuwang.base.common.enums.ReportType;
|
|
import com.simuwang.base.common.enums.ReportType;
|
|
|
|
+import com.simuwang.base.common.exception.ReportParseException;
|
|
import com.simuwang.base.common.util.EmailUtil;
|
|
import com.simuwang.base.common.util.EmailUtil;
|
|
import com.simuwang.base.common.util.ExcelUtil;
|
|
import com.simuwang.base.common.util.ExcelUtil;
|
|
import com.simuwang.base.common.util.FileUtil;
|
|
import com.simuwang.base.common.util.FileUtil;
|
|
@@ -21,7 +22,9 @@ import com.simuwang.base.pojo.dos.*;
|
|
import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
|
|
import com.simuwang.base.pojo.dto.EmailContentInfoDTO;
|
|
import com.simuwang.base.pojo.dto.EmailFundNavDTO;
|
|
import com.simuwang.base.pojo.dto.EmailFundNavDTO;
|
|
import com.simuwang.base.pojo.dto.MailboxInfoDTO;
|
|
import com.simuwang.base.pojo.dto.MailboxInfoDTO;
|
|
|
|
+import com.simuwang.base.pojo.dto.report.ParseResult;
|
|
import com.simuwang.base.pojo.dto.report.ReportData;
|
|
import com.simuwang.base.pojo.dto.report.ReportData;
|
|
|
|
+import com.simuwang.base.pojo.dto.report.ReportParseStatus;
|
|
import com.simuwang.base.pojo.dto.report.ReportParserParams;
|
|
import com.simuwang.base.pojo.dto.report.ReportParserParams;
|
|
import com.simuwang.base.pojo.valuation.CmValuationTableAttribute;
|
|
import com.simuwang.base.pojo.valuation.CmValuationTableAttribute;
|
|
import com.simuwang.daq.components.report.parser.ReportParser;
|
|
import com.simuwang.daq.components.report.parser.ReportParser;
|
|
@@ -178,7 +181,7 @@ public class EmailParseService {
|
|
emailId = saveEmailParseInfo(emailParseInfoDO);
|
|
emailId = saveEmailParseInfo(emailParseInfoDO);
|
|
|
|
|
|
// python 报告解析接口结果
|
|
// python 报告解析接口结果
|
|
- List<ReportData> dataList = ListUtil.list(false);
|
|
|
|
|
|
+ List<ParseResult<ReportData>> dataList = ListUtil.list(false);
|
|
for (Map.Entry<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavEntry : fileNameNavMap.entrySet()) {
|
|
for (Map.Entry<EmailContentInfoDTO, List<EmailFundNavDTO>> fileNameNavEntry : fileNameNavMap.entrySet()) {
|
|
// 保存邮件文件表
|
|
// 保存邮件文件表
|
|
EmailContentInfoDTO emailContentInfoDTO = fileNameNavEntry.getKey();
|
|
EmailContentInfoDTO emailContentInfoDTO = fileNameNavEntry.getKey();
|
|
@@ -198,10 +201,8 @@ public class EmailParseService {
|
|
}
|
|
}
|
|
if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) {
|
|
if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType)) {
|
|
// 解析结果(可以从python获取或者自行解析)并保存报告
|
|
// 解析结果(可以从python获取或者自行解析)并保存报告
|
|
- ReportData data = this.parseReportAndHandleResult(fileId, emailContentInfoDTO);
|
|
|
|
- if (data != null) {
|
|
|
|
- dataList.add(data);
|
|
|
|
- }
|
|
|
|
|
|
+ ParseResult<ReportData> parseResult = this.parseReportAndHandleResult(fileId, emailContentInfoDTO);
|
|
|
|
+ dataList.add(parseResult);
|
|
}
|
|
}
|
|
for (EmailFundNavDTO fundNavDTO : fundNavDTOList) {
|
|
for (EmailFundNavDTO fundNavDTO : fundNavDTOList) {
|
|
// 设置净值数据的解析状态
|
|
// 设置净值数据的解析状态
|
|
@@ -215,11 +216,6 @@ public class EmailParseService {
|
|
// 更新邮件解析结果 -> 当【净值日期】和【备案编码/基金名称】能正常解读,即识别为【成功】
|
|
// 更新邮件解析结果 -> 当【净值日期】和【备案编码/基金名称】能正常解读,即识别为【成功】
|
|
long successNavCount = fileNameNavMap.values().stream().flatMap(List::stream).filter(e -> e != null && StrUtil.isBlank(e.getFailReason())).count();
|
|
long successNavCount = fileNameNavMap.values().stream().flatMap(List::stream).filter(e -> e != null && StrUtil.isBlank(e.getFailReason())).count();
|
|
emailParseStatus = successNavCount >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
|
|
emailParseStatus = successNavCount >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
|
|
- // 报告邮件有一条成功就表示整体成功
|
|
|
|
- if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) && CollUtil.isNotEmpty(dataList)) {
|
|
|
|
- long count = dataList.size();
|
|
|
|
- emailParseStatus = count >= 1 ? EmailParseStatusConst.SUCCESS : EmailParseStatusConst.FAIL;
|
|
|
|
- }
|
|
|
|
String failReason = null;
|
|
String failReason = null;
|
|
if (emailParseStatus == EmailParseStatusConst.FAIL) {
|
|
if (emailParseStatus == EmailParseStatusConst.FAIL) {
|
|
// 邮件解析失败时 -> 保存失败原因
|
|
// 邮件解析失败时 -> 保存失败原因
|
|
@@ -227,6 +223,14 @@ public class EmailParseService {
|
|
List<EmailFundNavDTO> navDTOList = fileNameNavMap.values().stream().flatMap(List::stream).toList();
|
|
List<EmailFundNavDTO> navDTOList = fileNameNavMap.values().stream().flatMap(List::stream).toList();
|
|
failReason = hasPdfFile == 1 && CollUtil.isEmpty(navDTOList) ? "无法从pdf文件中获取到数据" : navDTOList.stream().map(EmailFundNavDTO::getFailReason).distinct().collect(Collectors.joining("/"));
|
|
failReason = hasPdfFile == 1 && CollUtil.isEmpty(navDTOList) ? "无法从pdf文件中获取到数据" : navDTOList.stream().map(EmailFundNavDTO::getFailReason).distinct().collect(Collectors.joining("/"));
|
|
}
|
|
}
|
|
|
|
+ // 报告邮件有一条失败就表示整个邮件解析失败
|
|
|
|
+ if (Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) && CollUtil.isNotEmpty(dataList)) {
|
|
|
|
+ failReason = dataList.stream().filter(e -> !Objects.equals(1, e.getStatus()))
|
|
|
|
+ .findFirst().map(ParseResult::getMsg).orElse(null);
|
|
|
|
+ if (failReason != null) {
|
|
|
|
+ emailParseStatus = EmailParseStatusConst.FAIL;
|
|
|
|
+ }
|
|
|
|
+ }
|
|
emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus, failReason);
|
|
emailParseInfoMapper.updateParseStatus(emailId, emailParseStatus, failReason);
|
|
}
|
|
}
|
|
|
|
|
|
@@ -364,11 +368,14 @@ public class EmailParseService {
|
|
}).collect(Collectors.toList());
|
|
}).collect(Collectors.toList());
|
|
}
|
|
}
|
|
|
|
|
|
- private ReportData parseReportAndHandleResult(int fileId, EmailContentInfoDTO emailContentInfoDTO) {
|
|
|
|
|
|
+ private ParseResult<ReportData> parseReportAndHandleResult(int fileId, EmailContentInfoDTO emailContentInfoDTO) {
|
|
|
|
+ ParseResult<ReportData> result = new ParseResult<>();
|
|
String fileName = emailContentInfoDTO.getFileName();
|
|
String fileName = emailContentInfoDTO.getFileName();
|
|
Integer emailType = emailContentInfoDTO.getEmailType();
|
|
Integer emailType = emailContentInfoDTO.getEmailType();
|
|
if (!Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) || StrUtil.isBlank(fileName)) {
|
|
if (!Objects.equals(EmailTypeConst.REPORT_EMAIL_TYPE, emailType) || StrUtil.isBlank(fileName)) {
|
|
- return null;
|
|
|
|
|
|
+ result.setStatus(ReportParseStatus.NOT_A_REPORT.getCode());
|
|
|
|
+ result.setMsg(ReportParseStatus.NOT_A_REPORT.getMsg());
|
|
|
|
+ return result;
|
|
}
|
|
}
|
|
Pattern pattern = Pattern.compile("S(?:[A-Z]{0}[0-9]{5}|[A-Z][0-9]{4}|[A-Z]{2}[0-9]{3}|[A-Z]{3}[0-9]{2})");
|
|
Pattern pattern = Pattern.compile("S(?:[A-Z]{0}[0-9]{5}|[A-Z][0-9]{4}|[A-Z]{2}[0-9]{3}|[A-Z]{3}[0-9]{2})");
|
|
Matcher matcher = pattern.matcher(fileName);
|
|
Matcher matcher = pattern.matcher(fileName);
|
|
@@ -401,8 +408,14 @@ public class EmailParseService {
|
|
.filepath(emailContentInfoDTO.getFilePath()).registerNumber(registerNumber).build();
|
|
.filepath(emailContentInfoDTO.getFilePath()).registerNumber(registerNumber).build();
|
|
ReportParser<ReportData> instance = this.reportParserFactory.getInstance(reportType, fileType);
|
|
ReportParser<ReportData> instance = this.reportParserFactory.getInstance(reportType, fileType);
|
|
reportData = instance.parse(params);
|
|
reportData = instance.parse(params);
|
|
|
|
+ } catch (ReportParseException e) {
|
|
|
|
+ log.error("报告{}解析失败\n{}", params, e.getMsg());
|
|
|
|
+ result.setStatus(e.getCode());
|
|
|
|
+ result.setMsg(e.getMsg());
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
log.error("报告{}解析失败\n{}", params, ExceptionUtil.stacktraceToString(e));
|
|
log.error("报告{}解析失败\n{}", params, ExceptionUtil.stacktraceToString(e));
|
|
|
|
+ result.setStatus(ReportParseStatus.PARSE_FAIL.getCode());
|
|
|
|
+ result.setMsg(StrUtil.format(ReportParseStatus.PARSE_FAIL.getMsg(), e.getMessage()));
|
|
} finally {
|
|
} finally {
|
|
parserWatch.stop();
|
|
parserWatch.stop();
|
|
if (log.isInfoEnabled()) {
|
|
if (log.isInfoEnabled()) {
|
|
@@ -425,7 +438,10 @@ public class EmailParseService {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
- return reportData;
|
|
|
|
|
|
+ result.setStatus(1);
|
|
|
|
+ result.setMsg("报告解析成功");
|
|
|
|
+ result.setData(reportData);
|
|
|
|
+ return result;
|
|
}
|
|
}
|
|
|
|
|
|
private void saveNavAndAssetNet(Integer fileId, List<EmailFundNavDTO> fundNavDTOList, Date parseDate) {
|
|
private void saveNavAndAssetNet(Integer fileId, List<EmailFundNavDTO> fundNavDTOList, Date parseDate) {
|