fix: 处理大数据分批入库的优化

service-daq/src/main/java/com/simuwang/daq/service/EmailParseService.java

@@ -479,7 +479,26 @@ public class EmailParseService {
         if (CollUtil.isNotEmpty(emailFundNavDOList)) {
             // 先删除文件id下的净值数据(考虑到重新解析的需求,如果是首次解析,那么file_id下不存在净值数据)
             emailFundNavMapper.deleteByFileId(fileId);
-            emailFundNavMapper.batchInsert(emailFundNavDOList);
+            try{
+                if(emailFundNavDOList.size() > 500){
+                    List<EmailFundNavDO> subEmailFundNavlist = new ArrayList<>();
+                    for (EmailFundNavDO emailFundNavDO : emailFundNavDOList) {
+                        subEmailFundNavlist.add(emailFundNavDO);
+                        if(subEmailFundNavlist.size() >=500){
+                            emailFundNavMapper.batchInsert(subEmailFundNavlist);
+                            subEmailFundNavlist.clear();
+                        }
+                    }
+                    if(subEmailFundNavlist.size() > 0){
+                        emailFundNavMapper.batchInsert(subEmailFundNavlist);
+                        subEmailFundNavlist.clear();
+                    }
+                }else{
+                    emailFundNavMapper.batchInsert(emailFundNavDOList);
+                }
+            }catch (Exception e){
+                log.error("插入邮件净值报错：文件ID:"+fileId+e.getMessage(),e);
+            }
             List<NavDO> navDOList = emailFundNavDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
                     .map(e -> BeanUtil.copyProperties(e, NavDO.class)).collect(Collectors.toList());
             saveNavDo(navDOList);
@@ -490,7 +509,27 @@ public class EmailParseService {
         if (CollUtil.isNotEmpty(emailFundAssetDOList)) {
             // 先删除file_id下的规模数据(考虑到重新解析的需求,如果是首次解析,那么file_id下不存在规模数据)
             emailFundAssetMapper.deleteByFileId(fileId);
-            emailFundAssetMapper.batchInsert(emailFundAssetDOList);
+            try{
+                if(emailFundAssetDOList.size() > 500){
+                    List<EmailFundAssetDO> subEmailFundAssetlist = new ArrayList<>();
+                    for (EmailFundAssetDO emailFundAssetDO : emailFundAssetDOList) {
+                        subEmailFundAssetlist.add(emailFundAssetDO);
+                        if(subEmailFundAssetlist.size() >=500){
+                            emailFundAssetMapper.batchInsert(subEmailFundAssetlist);
+                            subEmailFundAssetlist.clear();
+                        }
+                    }
+                    if(subEmailFundAssetlist.size() > 0){
+                        emailFundAssetMapper.batchInsert(subEmailFundAssetlist);
+                        subEmailFundAssetlist.clear();
+                    }
+                }else{
+                    emailFundAssetMapper.batchInsert(emailFundAssetDOList);
+                }
+
+            }catch (Exception e){
+                log.error("插入邮件净值报错：文件ID:"+fileId+e.getMessage(),e);
+            }
             List<AssetDO> assetDOList = emailFundAssetDOList.stream().filter(e -> StrUtil.isNotBlank(e.getFundId()))
                     .map(e -> BeanUtil.copyProperties(e, AssetDO.class)).collect(Collectors.toList());
             saveAssetDo(assetDOList);