fix: 数据缺失计算修复

service-base/src/main/java/com/simuwang/base/mapper/DeletionInfoMapper.java

@@ -47,4 +47,5 @@ public interface DeletionInfoMapper {
 
     void updateSendStatusByFundId(@Param("fundIdList")List<String> fundIdList);
 
+    void deleteDeletionRemark(@Param("fundId")String fundId, @Param("deletionType")Integer deletionType, @Param("tradeDate")String tradeDate);
 }

service-base/src/main/resources/mapper/DeletionInfoMapper.xml

@@ -51,6 +51,10 @@
             and deletion_date=#{deletionDate}
         </if>
     </update>
+    <delete id="deleteDeletionRemark">
+        update PPW_EMAIL.deletion_info  set isvalid=0,updatetime=sysdate() where isvalid=1 and fundId=#{fundId} and deletion_type=#{deletionType}
+        and deletion_date=#{tradeDate}
+    </delete>
     <select id="searchDeletionList" resultMap="BaseResultMap"
             parameterType="com.simuwang.base.pojo.dto.query.DeletionPageQuery">
         SELECT distinct

service-base/src/main/resources/mapper/EmailFileInfoMapper.xml

@@ -116,7 +116,13 @@
     </select>
     <select id="countFundFileInfo" resultType="java.lang.Long"
             parameterType="com.simuwang.base.pojo.dto.query.FundFilePageQuery">
-        SELECT count(DISTINCT efi.file_path)
+        select count(1) from (
+        SELECT DISTINCT
+            efn.fund_id as "fundId",
+            efn.fund_name as "fundName",
+            efi.file_name as "fileName",
+            efi.file_path as "filePath",
+            efi.createtime as "createTime"
         FROM
             PPW_EMAIL.EMAIL_FILE_INFO efi
                 JOIN PPW_EMAIL.EMAIL_FUND_NAV efn
@@ -126,7 +132,7 @@
         WHERE efi.isvalid = 1
           AND efn.isvalid = 1
           AND epi.isvalid = 1
-          AND epi.EMAIL_TYPE = #{fileType} and efn.fund_id=#{fundId}
+          AND epi.EMAIL_TYPE = #{fileType} and efn.fund_id=#{fundId}) a
     </select>
     <select id="countEmailDetailById" resultType="java.lang.Long"
             parameterType="com.simuwang.base.pojo.dto.query.EmailFileQuery">

service-manage/src/main/java/com/simuwang/manage/service/impl/DeletionServiceImpl.java

@@ -20,10 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -147,8 +144,6 @@ public class DeletionServiceImpl implements DeletionService {
         List<AssetDO> assetDOList = assetMapper.selectAssetByFundId(fundId);
         //查询成立日到今天为止的交易日集合
         List<TradeDateDO> tradeDateDOList = tradeDateMapper.selectTradeDate(inceptionDate,today);
-        //延迟天数为3天
-        tradeDateDOList = tradeDateDOList.subList(0,tradeDateDOList.size()-3);
         if(deletionDownParam.getDeletionType() != null && deletionDownParam.getDeletionType().equals(DeletionType.NAV_DELETION.getCode())){
             navDeletion(fundId,navDOList,tradeDateDOList,fundReportFrequencyDO);
         }else if(deletionDownParam.getDeletionType() != null && deletionDownParam.getDeletionType().equals(DeletionType.ASSET_DELETION.getCode())){
@@ -215,6 +210,13 @@ public class DeletionServiceImpl implements DeletionService {
                     deletionInfoMapper.updateRemark(fundId,DeletionType.ASSET_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
                     continue;
                 }
+                if(tradeDateDOList.size() <= 3){
+                    continue;
+                }
+                TradeDateDO tradeDateDO = tradeDateDOList.get(tradeDateDOList.size()-3);
+                if(tradeDateDO.getTradeDate().compareTo(DateUtils.parse(tradeDate,DateUtils.YYYY_MM_DD)) < 0){
+                    continue;
+                }
                 //写入缺失信息表
                 saveDeletionInfoDO(fundId,tradeDate,DeletionType.ASSET_DELETION.getCode());
             }
@@ -224,7 +226,7 @@ public class DeletionServiceImpl implements DeletionService {
             TreeMap<Integer,List<AssetDO>> weekNavListMap = new TreeMap<>();
             //按周数整合
             for(String priceDate : navListMap.keySet()){
-                Integer weekOfYear = DateUtil.weekOfYear(DateUtils.parse(priceDate,DateUtils.YYYY_MM_DD));
+                Integer weekOfYear = Integer.parseInt(priceDate.substring(0,4)+DateUtil.weekOfYear(DateUtils.parse(priceDate,DateUtils.YYYY_MM_DD)));
                 if(weekNavListMap.containsKey(weekOfYear)){
                     List<AssetDO> assetDOS = weekNavListMap.get(weekOfYear);
                     assetDOS.addAll(navListMap.get(priceDate));
@@ -235,13 +237,24 @@ public class DeletionServiceImpl implements DeletionService {
                     weekNavListMap.put(weekOfYear,navDOS);
                 }
             }
-            Map<Integer,List<TradeDateDO>> tradeListMap = tradeDateDOList.stream().collect(Collectors.groupingBy(e -> e.getWeekOfYear()));
+            Map<Integer,List<TradeDateDO>> tradeListMap = tradeDateDOList.stream().collect(Collectors.groupingBy(e -> e.getYearWeek()));
             for(Integer weekOfYear : tradeListMap.keySet()){
+                List<AssetDO> assetDOS = weekNavListMap.get(weekOfYear);
                 if(weekNavListMap.containsKey(weekOfYear)){
-                    List<AssetDO> assetDOS = weekNavListMap.get(weekOfYear);
-                    String tradeDate = DateUtils.format(assetDOS.get(assetDOS.size()-1).getPriceDate(),DateUtils.YYYY_MM_DD);
-                    deletionInfoMapper.updateRemark(fundId,DeletionType.ASSET_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
+                    for(AssetDO assetDO : assetDOS){
+                        String tradeDate = DateUtils.format(assetDO.getPriceDate(),DateUtils.YYYY_MM_DD);
+                        deletionInfoMapper.updateRemark(fundId,DeletionType.ASSET_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
+                    }
                     continue;
+                }else{
+                    if(StringUtil.isNotEmpty(assetDOS)){
+                        for(int idx=0;idx < assetDOS.size()-1 ; idx++){
+                            AssetDO assetDO = assetDOS.get(idx);
+                            String tradeDate = DateUtils.format(assetDO.getPriceDate(),DateUtils.YYYY_MM_DD);
+                            deletionInfoMapper.deleteDeletionRemark(fundId,DeletionType.ASSET_DELETION.getCode(),tradeDate);
+                        }
+                    }
+
                 }
                 //不包含的话，默认取每周的最后一个交易日作为周净值日期
                 List<TradeDateDO> tradeDateDOS = tradeListMap.get(weekOfYear);
@@ -249,6 +262,14 @@ public class DeletionServiceImpl implements DeletionService {
                 if(tradeDateDOS.size() > 0){
                     tradeDate = DateUtils.format(tradeDateDOS.get(tradeDateDOS.size()-1).getTradeDate(),DateUtils.YYYY_MM_DD);
                 }
+                //判断当前缺失交易日是否是近三个交易日
+                if(tradeDateDOList.size() <= 3){
+                    continue;
+                }
+                TradeDateDO tradeDateDO = tradeDateDOList.get(tradeDateDOList.size()-3);
+                if(tradeDateDO.getTradeDate().compareTo(DateUtils.parse(tradeDate,DateUtils.YYYY_MM_DD)) < 0){
+                    continue;
+                }
                 //写入缺失信息表
                 saveDeletionInfoDO(fundId,tradeDate,DeletionType.ASSET_DELETION.getCode());
             }
@@ -271,11 +292,27 @@ public class DeletionServiceImpl implements DeletionService {
             }
             Map<String,List<TradeDateDO>> tradeListMap = tradeDateDOList.stream().collect(Collectors.groupingBy(e -> e.getYearmonth()));
             for(String yearMonth : tradeListMap.keySet()){
+                //本月的数据不考虑
+                String thisMonth = DateUtils.format(new Date(),DateUtils.YYYY_MM);
+                if(yearMonth.equals(thisMonth)){
+                    continue;
+                }
+                List<AssetDO> assetDOS = monthNavListMap.get(yearMonth);
                 if(monthNavListMap.containsKey(yearMonth)){
-                    List<AssetDO> assetDOS = monthNavListMap.get(yearMonth);
-                    String tradeDate = DateUtils.format(assetDOS.get(assetDOS.size()-1).getPriceDate(),DateUtils.YYYY_MM_DD);
-                    deletionInfoMapper.updateRemark(fundId,DeletionType.ASSET_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
+                    for(AssetDO assetDO : assetDOS){
+                        String tradeDate = DateUtils.format(assetDO.getPriceDate(),DateUtils.YYYY_MM_DD);
+                        deletionInfoMapper.updateRemark(fundId,DeletionType.ASSET_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
+                    }
                     continue;
+                }else{
+                    if(StringUtil.isNotEmpty(assetDOS)){
+                        for(int idx=0;idx < assetDOS.size()-1 ; idx++){
+                            AssetDO assetDO = assetDOS.get(idx);
+                            String tradeDate = DateUtils.format(assetDO.getPriceDate(),DateUtils.YYYY_MM_DD);
+                            deletionInfoMapper.deleteDeletionRemark(fundId,DeletionType.ASSET_DELETION.getCode(),tradeDate);
+                        }
+                    }
+
                 }
                 //不包含的话，默认取每周的最后一个交易日作为周净值日期
                 List<TradeDateDO> tradeDateDOS = tradeListMap.get(yearMonth);
@@ -283,6 +320,13 @@ public class DeletionServiceImpl implements DeletionService {
                 if(tradeDateDOS.size() > 0){
                     tradeDate = DateUtils.format(tradeDateDOS.get(tradeDateDOS.size()-1).getTradeDate(),DateUtils.YYYY_MM_DD);
                 }
+                if(tradeDateDOList.size() <= 3){
+                    continue;
+                }
+                TradeDateDO tradeDateDO = tradeDateDOList.get(tradeDateDOList.size()-3);
+                if(tradeDateDO.getTradeDate().compareTo(DateUtils.parse(tradeDate,DateUtils.YYYY_MM_DD)) < 0){
+                    continue;
+                }
                 //写入缺失信息表
                 saveDeletionInfoDO(fundId,tradeDate,DeletionType.ASSET_DELETION.getCode());
             }
@@ -308,7 +352,7 @@ public class DeletionServiceImpl implements DeletionService {
             TreeMap<Integer,List<NavDO>> weekNavListMap = new TreeMap<>();
             //按周数整合
             for(String priceDate : navListMap.keySet()){
-                Integer weekOfYear = DateUtil.weekOfYear(DateUtils.parse(priceDate,DateUtils.YYYY_MM_DD));
+                Integer weekOfYear = Integer.parseInt(priceDate.substring(0,4)+DateUtil.weekOfYear(DateUtils.parse(priceDate,DateUtils.YYYY_MM_DD)));;
                 if(weekNavListMap.containsKey(weekOfYear)){
                     List<NavDO> navDOS = weekNavListMap.get(weekOfYear);
                     navDOS.addAll(navListMap.get(priceDate));
@@ -319,13 +363,24 @@ public class DeletionServiceImpl implements DeletionService {
                     weekNavListMap.put(weekOfYear,navDOS);
                 }
             }
-            Map<Integer,List<TradeDateDO>> tradeListMap = tradeDateDOList.stream().collect(Collectors.groupingBy(e -> e.getWeekOfYear()));
+            Map<Integer,List<TradeDateDO>> tradeListMap = tradeDateDOList.stream().collect(Collectors.groupingBy(e -> e.getYearWeek()));
             for(Integer weekOfYear : tradeListMap.keySet()){
+                List<NavDO> navDOS = weekNavListMap.get(weekOfYear);
                 if(weekNavListMap.containsKey(weekOfYear)){
-                    List<NavDO> navDOS = weekNavListMap.get(weekOfYear);
-                    String tradeDate = DateUtils.format(navDOS.get(navDOS.size()-1).getPriceDate(),DateUtils.YYYY_MM_DD);
-                    deletionInfoMapper.updateRemark(fundId,DeletionType.NAV_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
+                    for(NavDO navDO : navDOS){
+                        String tradeDate = DateUtils.format(navDO.getPriceDate(),DateUtils.YYYY_MM_DD);
+                        deletionInfoMapper.updateRemark(fundId,DeletionType.NAV_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
+                    }
                     continue;
+                }else{
+                    //防止频率变更导致数据异常，需要把当前日期下所在的频率全部无效掉
+                    if(StringUtil.isNotEmpty(navDOS)){
+                        for(int idx=0;idx < navDOS.size()-1 ; idx++){
+                            NavDO navDO = navDOS.get(idx);
+                            String tradeDate = DateUtils.format(navDO.getPriceDate(),DateUtils.YYYY_MM_DD);
+                            deletionInfoMapper.deleteDeletionRemark(fundId,DeletionType.NAV_DELETION.getCode(),tradeDate);
+                        }
+                    }
                 }
                 //不包含的话，默认取每周的最后一个交易日作为周净值日期
                 List<TradeDateDO> tradeDateDOS = tradeListMap.get(weekOfYear);
@@ -333,6 +388,13 @@ public class DeletionServiceImpl implements DeletionService {
                 if(tradeDateDOS.size() > 0){
                     tradeDate = DateUtils.format(tradeDateDOS.get(tradeDateDOS.size()-1).getTradeDate(),DateUtils.YYYY_MM_DD);
                 }
+                if(tradeDateDOList.size() <= 3){
+                    continue;
+                }
+                TradeDateDO tradeDateDO = tradeDateDOList.get(tradeDateDOList.size()-3);
+                if(tradeDateDO.getTradeDate().compareTo(DateUtils.parse(tradeDate,DateUtils.YYYY_MM_DD)) < 0){
+                    continue;
+                }
                 //写入缺失信息表
                 saveDeletionInfoDO(fundId,tradeDate,DeletionType.NAV_DELETION.getCode());
             }
@@ -355,11 +417,27 @@ public class DeletionServiceImpl implements DeletionService {
             }
             Map<String,List<TradeDateDO>> tradeListMap = tradeDateDOList.stream().collect(Collectors.groupingBy(e -> e.getYearmonth()));
             for(String yearMonth : tradeListMap.keySet()){
+                //本月的数据不考虑
+                String thisMonth = DateUtils.format(new Date(),DateUtils.YYYY_MM);
+                if(yearMonth.equals(thisMonth)){
+                    continue;
+                }
+                List<NavDO> navDOS = monthNavListMap.get(yearMonth);
                 if(monthNavListMap.containsKey(yearMonth)){
-                    List<NavDO> navDOS = monthNavListMap.get(yearMonth);
-                    String tradeDate = DateUtils.format(navDOS.get(navDOS.size()-1).getPriceDate(),DateUtils.YYYY_MM_DD);
-                    deletionInfoMapper.updateRemark(fundId,DeletionType.NAV_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
+                    for(NavDO navDO : navDOS){
+                        String tradeDate = DateUtils.format(navDO.getPriceDate(),DateUtils.YYYY_MM_DD);
+                        deletionInfoMapper.updateRemark(fundId,DeletionType.NAV_DELETION.getCode(),tradeDate,DeletionType.NO_DELETION.getInfo(),null);
+                    }
                     continue;
+                }else{
+                    //防止频率变更导致数据异常，需要把当前日期下所在的频率全部无效掉
+                    if(StringUtil.isNotEmpty(navDOS)){
+                        for(NavDO navDO : navDOS){
+                            String tradeDate = DateUtils.format(navDO.getPriceDate(),DateUtils.YYYY_MM_DD);
+                            deletionInfoMapper.deleteDeletionRemark(fundId,DeletionType.NAV_DELETION.getCode(),tradeDate);
+                        }
+                    }
+
                 }
                 //不包含的话，默认取每周的最后一个交易日作为周净值日期
                 List<TradeDateDO> tradeDateDOS = tradeListMap.get(yearMonth);
@@ -367,6 +445,13 @@ public class DeletionServiceImpl implements DeletionService {
                 if(tradeDateDOS.size() > 0){
                     tradeDate = DateUtils.format(tradeDateDOS.get(tradeDateDOS.size()-1).getTradeDate(),DateUtils.YYYY_MM_DD);
                 }
+                if(tradeDateDOList.size() <= 3){
+                    continue;
+                }
+                TradeDateDO tradeDateDO = tradeDateDOList.get(tradeDateDOList.size()-3);
+                if(tradeDateDO.getTradeDate().compareTo(DateUtils.parse(tradeDate,DateUtils.YYYY_MM_DD)) < 0){
+                    continue;
+                }
                 //写入缺失信息表
                 saveDeletionInfoDO(fundId,tradeDate,DeletionType.NAV_DELETION.getCode());
             }

service-manage/src/main/java/com/simuwang/manage/task/FundDeletionTask.java

@@ -33,10 +33,13 @@ public class FundDeletionTask {
     private NavMapper navMapper;
     @Autowired
     private DeletionService deletionService;
-    @Scheduled(cron = "0 30 5,10,19 * * ?")
+    @Scheduled(cron = "0 0 5,12,19 * * ?")
     public void computeDeletion(){
         List<String> fundIdList = navMapper.getAllFundId();
         for(String fundId : fundIdList){
+//            if(!fundId.contains("HF00002FDW")){
+//               continue;
+//            }
             DeletionDownParam deletionDownParam = new DeletionDownParam();
             deletionDownParam.setFundId(fundId);
             deletionService.computeDeletion(deletionDownParam);

service-manage/src/main/java/com/simuwang/manage/task/SendCompanyEmailTask.java

@@ -28,7 +28,7 @@ public class SendCompanyEmailTask {
 
     @Autowired
     private CompanyEmailConfigService companyEmailConfigService;
-//    @Scheduled(cron = "0 0/3 * * * ?")
+    @Scheduled(cron = "0 09 15 * * ?")
     public void sendEmail(){
         List<CompanyEmailConfigDO> configDOList = companyEmailConfigMapper.getAllCompanyConfig();
         Map<String,List<CompanyEmailConfigDO>> companyListMap = configDOList.stream().collect(Collectors.groupingBy(e -> e.getCompanyId()));