重构json转dto的转换器

service-daq/src/main/java/com/simuwang/daq/components/PythonReportConverter.java

@@ -2,14 +2,23 @@ package com.simuwang.daq.components;
 
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.util.ReflectUtil;
+import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONArray;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
+import com.simuwang.base.pojo.dos.report.BaseReportDO;
 import com.simuwang.base.pojo.dto.report.*;
 
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 
+/**
+ * @author wangzaijun
+ * @date 2024/9/27 16:06
+ * @description 自定义的python报告解析结果转换器，json -> dto
+ */
 public class PythonReportConverter {
     @SuppressWarnings("unchecked")
     public static <T extends ReportData> PythonResult<T> convert(JSONObject jsonObject, Integer type) {
@@ -36,200 +45,106 @@ public class PythonReportConverter {
 
     private static MonthlyReportData convertMonthly(JSONObject jsonObject) {
         MonthlyReportData reportData = new MonthlyReportData();
-        reportData.setBaseInfo(convertBaseInfo(jsonObject));
-        reportData.setFundInfo(convertFundInfo(jsonObject));
-        reportData.setNetReport(convertNetReport(jsonObject));
+        reportData.setBaseInfo(convertToObj(jsonObject, "base_info", ReportBaseInfoDTO.class));
+        reportData.setFundInfo(convertToObj(jsonObject, "fund_info", ReportFundInfoDTO.class));
+        reportData.setNetReport(convertToList(jsonObject, "net_report", ReportNetReportDTO.class));
         return reportData;
     }
 
     private static QuarterlyReportData convertQuarterly(JSONObject jsonObject) {
         QuarterlyReportData reportData = new QuarterlyReportData();
-        reportData.setBaseInfo(convertBaseInfo(jsonObject));
-        reportData.setFundInfo(convertFundInfo(jsonObject));
-        reportData.setAssetAllocation(convertAssetAllocation(jsonObject));
-        reportData.setFinancialIndicators(convertFinancialIndicators(jsonObject));
-        reportData.setInvestmentIndustry(convertInvestmentIndustry(jsonObject));
-        reportData.setShareChange(convertShareChange(jsonObject));
+        reportData.setBaseInfo(convertToObj(jsonObject, "base_info", ReportBaseInfoDTO.class));
+        reportData.setFundInfo(convertToObj(jsonObject, "fund_info", ReportFundInfoDTO.class));
+        reportData.setAssetAllocation(convertToList(jsonObject, "asset_allocation", ReportAssetAllocationDTO.class));
+        reportData.setFinancialIndicators(convertToList(jsonObject, "financial_indicators", ReportFinancialIndicatorsDTO.class));
+        reportData.setInvestmentIndustry(convertToList(jsonObject, "investment_industry", ReportInvestmentIndustryDTO.class));
+        reportData.setShareChange(convertToList(jsonObject, "share_change", ReportShareChangeDTO.class));
         return reportData;
     }
 
-    private static ReportBaseInfoDTO convertBaseInfo(JSONObject jsonObject) {
-        ReportBaseInfoDTO baseInfo = new ReportBaseInfoDTO();
-        if (jsonObject == null || !jsonObject.containsKey("base_info")) {
-            return baseInfo;
-        }
-        JSONArray jsonArray = jsonObject.getJSONArray("base_info");
-        JSONObject obj = jsonArray.getJSONObject(0);
-        if (obj == null || obj.isEmpty()) {
-            return baseInfo;
-        }
-        baseInfo.setFileId(obj.getInt("file_id"));
-        baseInfo.setReportName(obj.getStr("report_name"));
-        baseInfo.setReportType(obj.getStr("report_type"));
-        baseInfo.setReportDate(obj.getStr("report_date"));
-        return baseInfo;
+    /**
+     * json转集合后取第一个数据
+     *
+     * @param jsonObject 待获取的json对象
+     * @param key        字段
+     * @param clazz      对象类型class
+     * @param <T>        泛型
+     * @return /
+     */
+    private static <T extends BaseReportDTO<? extends BaseReportDO>> T convertToObj(JSONObject jsonObject, String key, Class<T> clazz) {
+        List<T> dtos = convertToList(jsonObject, key, clazz);
+        return CollUtil.isEmpty(dtos) ? null : dtos.get(0);
     }
 
-    private static ReportFundInfoDTO convertFundInfo(JSONObject jsonObject) {
-        ReportFundInfoDTO dto = new ReportFundInfoDTO();
-        if (jsonObject == null || !jsonObject.containsKey("fund_info")) {
-            return dto;
-        }
-        JSONArray jsonArray = jsonObject.getJSONArray("fund_info");
-        JSONObject obj = jsonArray.getJSONObject(0);
-        if (obj == null || obj.isEmpty()) {
-            return dto;
-        }
-        dto.setFileId(obj.getInt("file_id"));
-        dto.setAdvisorName(obj.getStr("advisor_name"));
-        dto.setCustodianName(obj.getStr("custodian_name"));
-        dto.setFundManager(obj.getStr("fund_manager"));
-        dto.setFundName(obj.getStr("fund_name"));
-        dto.setFundStrategyDescription(obj.getStr("fund_strategy_description"));
-        dto.setInceptionDate(obj.getStr("inception_date"));
-        dto.setIndustryTrend(obj.getStr("industry_trend"));
-        dto.setInvestmentObjective(obj.getStr("investment_objective"));
-        dto.setLeverage(obj.getBigDecimal("leverage"));
-        dto.setLeverageNote(obj.getStr("leverage_note"));
-        dto.setOperationType(obj.getStr("operation_type"));
-        dto.setRegisterNumber(obj.getStr("register_number"));
-        dto.setRiskReturnDesc(obj.getStr("risk_return_desc"));
-        dto.setSecondaryBenchmark(obj.getStr("secondary_benchmark"));
-        dto.setTrustName(obj.getStr("trust_name"));
-        dto.setDueDate(obj.getStr("due_date"));
-        dto.setIsReviewed(obj.getInt("is_reviewed"));
-        return dto;
-    }
-
-    private static List<ReportNetReportDTO> convertNetReport(JSONObject jsonObject) {
-        List<ReportNetReportDTO> dtos = ListUtil.list(false);
-        if (jsonObject == null || !jsonObject.containsKey("net_report")) {
-            return dtos;
-        }
-        JSONArray jsonArray = jsonObject.getJSONArray("net_report");
-        if (CollUtil.isEmpty(jsonArray)) {
+    /**
+     * json转换为集合
+     *
+     * @param jsonObject 待获取的json对象
+     * @param key        字段
+     * @param clazz      对象类型class
+     * @param <T>        泛型
+     * @return /
+     */
+    private static <T extends BaseReportDTO<? extends BaseReportDO>> List<T> convertToList(JSONObject jsonObject, String key, Class<T> clazz) {
+        List<T> dtos = ListUtil.list(false);
+        if (jsonObject == null || !jsonObject.containsKey(key)) {
             return dtos;
         }
-        for (Object o : jsonArray) {
-            JSONObject obj = JSONUtil.parseObj(o);
-            ReportNetReportDTO dto = new ReportNetReportDTO();
-            if (obj == null || obj.isEmpty()) {
-                continue;
-            }
-            dto.setFileId(obj.getInt("file_id"));
-            dto.setValuationDate(obj.getStr("valuation_date"));
-            dto.setFundAssetSize(obj.getBigDecimal("fund_asset_size"));
-            dto.setEndTotalShares(obj.getBigDecimal("end_total_shares"));
-            dto.setCumulativeNav(obj.getBigDecimal("cumulative_nav"));
-            dto.setNav(obj.getBigDecimal("nav"));
-            dtos.add(dto);
-        }
-        return dtos;
+        JSONArray jsonArray = jsonObject.getJSONArray(key);
+        return convertDTOs(jsonArray, clazz);
     }
 
-    private static List<ReportShareChangeDTO> convertShareChange(JSONObject jsonObject) {
-        List<ReportShareChangeDTO> dtos = ListUtil.list(false);
-        if (jsonObject == null || !jsonObject.containsKey("share_change")) {
-            return dtos;
-        }
-        JSONArray jsonArray = jsonObject.getJSONArray("share_change");
+    /**
+     * 通过反射绑定对象
+     *
+     * @param jsonArray 数据集合
+     * @param clazz     对象class对象
+     * @param <T>       泛型对象
+     * @return /
+     */
+    private static <T extends BaseReportDTO<? extends BaseReportDO>> List<T> convertDTOs(JSONArray jsonArray, Class<T> clazz) {
+        List<T> dtos = ListUtil.list(false);
         if (CollUtil.isEmpty(jsonArray)) {
             return dtos;
         }
         for (Object o : jsonArray) {
             JSONObject obj = JSONUtil.parseObj(o);
-            ReportShareChangeDTO dto = new ReportShareChangeDTO();
             if (obj == null || obj.isEmpty()) {
                 continue;
             }
-            dto.setFileId(obj.getInt("file_id"));
-            dto.setLevel(obj.getStr("level"));
-            dto.setInitTotalShares(obj.getBigDecimal("init_total_shares"));
-            dto.setRedemption(obj.getBigDecimal("redemption"));
-            dto.setSharePerAsset(obj.getBigDecimal("share_per_asset"));
-            dto.setSubscription(obj.getBigDecimal("subscription"));
-            dto.setSplit(obj.getBigDecimal("split"));
-            dtos.add(dto);
-        }
-        return dtos;
-    }
-
-    private static List<ReportAssetAllocationDTO> convertAssetAllocation(JSONObject jsonObject) {
-        List<ReportAssetAllocationDTO> dtos = ListUtil.list(false);
-        if (jsonObject == null || !jsonObject.containsKey("asset_allocation")) {
-            return dtos;
-        }
-        JSONArray jsonArray = jsonObject.getJSONArray("asset_allocation");
-        if (CollUtil.isEmpty(jsonArray)) {
-            return dtos;
-        }
-        for (Object o : jsonArray) {
-            JSONObject obj = JSONUtil.parseObj(o);
-            ReportAssetAllocationDTO dto = new ReportAssetAllocationDTO();
-            if (obj == null || obj.isEmpty()) {
-                continue;
+            T dto = setFieldValues(obj, clazz);
+            if (dto != null) {
+                dtos.add(dto);
             }
-            dto.setFileId(obj.getInt("file_id"));
-            dto.setAssetType(obj.getStr("asset_type"));
-            dto.setColumnName(obj.getStr("column_name"));
-            dto.setMarketValue(obj.getBigDecimal("market_value"));
-            dto.setRemark(obj.getStr("remark"));
-            dtos.add(dto);
         }
         return dtos;
     }
 
-    private static List<ReportFinancialIndicatorsDTO> convertFinancialIndicators(JSONObject jsonObject) {
-        List<ReportFinancialIndicatorsDTO> dtos = ListUtil.list(false);
-        if (jsonObject == null || !jsonObject.containsKey("financial_indicators")) {
-            return dtos;
-        }
-        JSONArray jsonArray = jsonObject.getJSONArray("financial_indicators");
-        if (CollUtil.isEmpty(jsonArray)) {
-            return dtos;
-        }
-        for (Object o : jsonArray) {
-            JSONObject obj = JSONUtil.parseObj(o);
-            ReportFinancialIndicatorsDTO dto = new ReportFinancialIndicatorsDTO();
-            if (obj == null || obj.isEmpty()) {
-                continue;
+    /**
+     * 通过反射设置对象的字段值
+     *
+     * @param jsonObject json对象
+     * @param clazz      对象class类型
+     * @param <T>        泛型对象
+     * @return 如果构建起初始化报错直接返回null，如果没有字段则跳过该字段的设置
+     */
+    private static <T extends BaseReportDTO<? extends BaseReportDO>> T setFieldValues(JSONObject jsonObject, Class<T> clazz) {
+        T obj = null;
+        try {
+            obj = clazz.getDeclaredConstructor().newInstance();
+        } catch (Exception ignored) {
+        }
+        if (obj == null) {
+            return null;
+        }
+        Set<String> columns = jsonObject.keySet();
+        for (String column : columns) {
+            String fieldName = StrUtil.toCamelCase(column);
+            try {
+                ReflectUtil.setFieldValue(obj, fieldName, jsonObject.get(column));
+            } catch (Exception ignored) {
             }
-            dto.setFileId(obj.getInt("file_id"));
-            dto.setLevel(obj.getStr("level"));
-            dto.setEndDate(obj.getInt("end_date"));
-            dto.setFundAssetSize(obj.getBigDecimal("fund_asset_aize"));
-            dto.setNav(obj.getBigDecimal("nav"));
-            dto.setProfit(obj.getBigDecimal("profit"));
-            dto.setRealizedIncome(obj.getBigDecimal("realized_income"));
-            dtos.add(dto);
         }
-        return dtos;
-    }
-
-    private static List<ReportInvestmentIndustryDTO> convertInvestmentIndustry(JSONObject jsonObject) {
-        List<ReportInvestmentIndustryDTO> dtos = ListUtil.list(false);
-        if (jsonObject == null || !jsonObject.containsKey("investment_industry")) {
-            return dtos;
-        }
-        JSONArray jsonArray = jsonObject.getJSONArray("investment_industry");
-        if (CollUtil.isEmpty(jsonArray)) {
-            return dtos;
-        }
-        for (Object o : jsonArray) {
-            JSONObject obj = JSONUtil.parseObj(o);
-            ReportInvestmentIndustryDTO dto = new ReportInvestmentIndustryDTO();
-            if (obj == null || obj.isEmpty()) {
-                continue;
-            }
-            dto.setFileId(obj.getInt("file_id"));
-            dto.setIndustryCode(obj.getStr("industry_code"));
-            dto.setIndustryName(obj.getStr("industry_name"));
-            dto.setInvestType(obj.getInt("invest_type"));
-            dto.setIsbCode(obj.getStr("isb_code"));
-            dto.setMarketValue(obj.getBigDecimal("market_value"));
-            dto.setRatio(obj.getBigDecimal("ratio"));
-            dtos.add(dto);
-        }
-        return dtos;
+        return obj;
     }
 }

service-daq/src/main/java/com/simuwang/daq/components/writer/AbstractReportWriter.java

@@ -30,6 +30,22 @@ public abstract class AbstractReportWriter<T extends ReportData> implements Repo
         StopWatch stopWatch = new StopWatch();
         stopWatch.start();
         // 基本信息+基金信息保存
+        this.saveBaseInfo(reportData);
+        this.saveFundInfo(reportData);
+        try {
+            // 其他信息保存
+            this.writeExtData(reportData);
+        } catch (Exception e) {
+            this.logger.error("报告解析结果之类型特有数据保存报错\n{}", ExceptionUtil.stacktraceToString(e));
+        }
+        stopWatch.stop();
+        long totalTimeMillis = stopWatch.getTotalTimeMillis();
+        if (this.logger.isInfoEnabled()) {
+            this.logger.info("报告解析结果保存成功，耗时：{}ms", totalTimeMillis);
+        }
+    }
+
+    private void saveBaseInfo(T reportData) {
         try {
             ReportBaseInfoDTO baseInfo = reportData.getBaseInfo();
             if (baseInfo != null) {
@@ -39,6 +55,9 @@ public abstract class AbstractReportWriter<T extends ReportData> implements Repo
         } catch (Exception e) {
             this.logger.error("报告解析结果之报告基本信息保存报错\n{}", ExceptionUtil.stacktraceToString(e));
         }
+    }
+
+    private void saveFundInfo(T reportData) {
         try {
             ReportFundInfoDTO fundInfo = reportData.getFundInfo();
             if (fundInfo != null) {
@@ -48,17 +67,6 @@ public abstract class AbstractReportWriter<T extends ReportData> implements Repo
         } catch (Exception e) {
             this.logger.error("报告解析结果之基金信息保存报错\n{}", ExceptionUtil.stacktraceToString(e));
         }
-        try {
-            // 其他信息保存
-            this.writeExtData(reportData);
-        } catch (Exception e) {
-            this.logger.error("报告解析结果之类型特有数据保存报错\n{}", ExceptionUtil.stacktraceToString(e));
-        }
-        stopWatch.stop();
-        long totalTimeMillis = stopWatch.getTotalTimeMillis();
-        if (this.logger.isInfoEnabled()) {
-            this.logger.info("报告解析结果保存成功，耗时：{}ms", totalTimeMillis);
-        }
     }
 
     protected abstract void writeExtData(T reportData);

service-daq/src/main/java/com/simuwang/daq/utils/ReportParseUtil.java

@@ -1,44 +1,23 @@
 package com.simuwang.daq.utils;
 
-import cn.hutool.core.collection.CollUtil;
-import cn.hutool.core.collection.ListUtil;
 import cn.hutool.core.map.MapUtil;
-import cn.hutool.core.util.ReflectUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.http.HttpUtil;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
 import com.simuwang.base.pojo.dto.report.PythonResult;
-import com.simuwang.daq.components.CustomPDFTextStripper;
 import com.simuwang.daq.components.PythonReportConverter;
-import com.simuwang.daq.dto.ReportFundInfo;
-import com.smppw.common.pojo.ValueLabelVO;
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.contentstream.PDFStreamEngine;
-import org.apache.pdfbox.contentstream.operator.text.ShowText;
 import org.apache.pdfbox.cos.COSName;
-import org.apache.pdfbox.io.RandomAccessReadBufferedFile;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDResources;
 import org.apache.pdfbox.pdmodel.common.PDStream;
-import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureTreeRoot;
 import org.apache.pdfbox.pdmodel.graphics.PDXObject;
 import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
-import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
-import org.apache.pdfbox.text.PDFTextStripper;
-import org.apache.pdfbox.text.PDFTextStripperByArea;
-import org.apache.pdfbox.text.TextPosition;
-import org.apache.pdfbox.util.Matrix;
-import technology.tabula.*;
-import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
 
-import java.awt.geom.Rectangle2D;
 import java.io.IOException;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 public class ReportParseUtil {
     public static void main(String[] args) throws IOException {