fix: CN-1313 系统内置知识库页导入更新文件更新Psiphon3 VPN，内容导入接口响应时间过长

2 files changed, 57 insertions, 42 deletions

diff --git a/cn-admin/src/main/java/net/geedge/common/utils/CSVUtils.java b/cn-admin/src/main/java/net/geedge/common/utils/CSVUtils.java
index d207577..fad27cb 100644
--- a/cn-admin/src/main/java/net/geedge/common/utils/CSVUtils.java
+++ b/cn-admin/src/main/java/net/geedge/common/utils/CSVUtils.java
@@ -2,6 +2,7 @@ package net.geedge.common.utils;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
 import java.util.UUID;
 import org.apache.commons.io.FileUtils;
@@ -34,12 +35,12 @@ public class CSVUtils {
         return data;
     }
 
-    public static boolean validateCSVHeader(CsvRow headers, List<String> headerInfos) {
+    public static boolean validateCSVHeader(List<String> headers, List<String> headerInfos) {
     	// 先判断表头数量是否一致  再循环判断每个表头是否存在 表头位置不用一一对应 存在即可
     	if(headers.size() == headerInfos.size()) {
     		for (int i = 0; i < headerInfos.size(); i++) {
                 String header = headers.get(i);
-                if(header.indexOf("\uFEFF") >= 0)
+                if(header.indexOf("\uFEFF")>= 0)
                     headers.set(i,header.substring(1)); //去除BOM头
                 if (!headerInfos.contains(header)) {
                     return false;
diff --git a/cn-admin/src/main/java/net/geedge/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java b/cn-admin/src/main/java/net/geedge/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java
index c4d9c7a..37ee7da 100644
--- a/cn-admin/src/main/java/net/geedge/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java
+++ b/cn-admin/src/main/java/net/geedge/modules/knowledge/service/impl/KnowledgeBaseServiceImpl.java
@@ -19,6 +19,7 @@ import static net.geedge.common.utils.Constant.LINK_DIRECTION_BUILT_IN;
 import static net.geedge.common.utils.Constant.PSIPHON3_IP;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -28,6 +29,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.UUID;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -47,12 +49,15 @@ import com.baomidou.mybatisplus.core.toolkit.ObjectUtils;
 import com.baomidou.mybatisplus.core.toolkit.StringPool;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 
+import cn.hutool.core.collection.ListUtil;
 import cn.hutool.core.io.CharsetDetector;
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.resource.ResourceUtil;
 import cn.hutool.core.text.csv.CsvData;
+import cn.hutool.core.text.csv.CsvReader;
 import cn.hutool.core.text.csv.CsvRow;
 import cn.hutool.core.text.csv.CsvUtil;
+import cn.hutool.core.util.CharsetUtil;
 import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.log.Log;
@@ -1035,16 +1040,16 @@ public class KnowledgeBaseServiceImpl extends ServiceImpl<KnowledgeBaseDao, Know
 		}
 		List<Integer> deleteDataIds = Lists.newArrayList();
 		List<Map<String, String>> saveOrUpdateList = Lists.newArrayList();
+		//覆盖操作时 将对应表中数据删除清空
+		if (StrUtil.equals(action, "overwrite")) {
+			basicImportAndExportService.removeList(knowledgeBase.getSource(), knowledgeBase.getKnowledgeId());
+		}
 		this.validAndBuildDatasets(file, knowledgeBase, action, saveOrUpdateList, deleteDataIds);
 		//更新操作时 删除多余重复数据
 		if (StrUtil.equals(action, "update") && !deleteDataIds.isEmpty()) {
 			basicImportAndExportService.removeList(knowledgeBase.getSource(), deleteDataIds);
 		}
-		//覆盖操作时 将对应表中数据删除清空
-		if (StrUtil.equals(action, "overwrite")) {
-			basicImportAndExportService.removeList(knowledgeBase.getSource(), knowledgeBase.getKnowledgeId());
-		}
-		this.saveOrUpdateBatch(saveOrUpdateList, knowledgeBase);
+//		this.saveOrUpdateBatch(saveOrUpdateList, knowledgeBase);
 		KnowledgeBaseSyncVersion syncVersion = knowledgeBaseSyncVersionService.syncVersion(knowledgeBase.getKnowledgeId());
 		KnowledgeBaseAuditLog log = new KnowledgeBaseAuditLog();
 		log.setDescription(knowledgeBase.getDescription());
@@ -1081,60 +1086,61 @@ public class KnowledgeBaseServiceImpl extends ServiceImpl<KnowledgeBaseDao, Know
 
         switch (fileType) {
             case "csv": {
-                CsvData csvData = CSVUtils.getCsvDataFromMultipartFile(file);
-                List<CsvRow> rows = csvData.getRows();
-                CsvRow headers = rows.get(0);
-                // 表头校验
-                if (!CSVUtils.validateCSVHeader(headers, headerInfos)) {
+            	File tempCsvFile = new File(Constant.TEMP_PATH + File.separator + "csv_" + UUID.randomUUID());
+            	try {
+					FileUtils.copyInputStreamToFile(file.getInputStream(), tempCsvFile);
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+    		    CsvReader reader = CsvUtil.getReader(ResourceUtil.getReader(tempCsvFile.getPath(), CharsetUtil.CHARSET_UTF_8));
+    		    reader.setContainsHeader(true);
+    		    Set<String> header = reader.stream().findFirst().get().getFieldMap().keySet();
+    		    ArrayList<String> headerInfo = ListUtil.toList(header);
+    		    // 表头校验
+                if (!CSVUtils.validateCSVHeader(headerInfo, headerInfos)) {
                     throw new CNException(RCode.EXCELFILE_HEADER_TEMPLATE_ERROR);
                 }
-                // 表头以及下标信息整理
-                Map<Integer,String> indexHeader = new HashMap<Integer,String>();
-                for(int i = 0;i<headers.size();i++) {
-                	indexHeader.put(i, headers.get(i));
-                }
-                for (int i = 1; i < rows.size(); i++) {
-                	boolean ipHandler = false;
-                    CsvRow csvRow = rows.get(i);
-                    Map<String, String> tmpMap = Tool.MapUtil.newHashMap();
-                    for (int j = 0; j < headerInfos.size(); j++) {
-                        String cellStr = StrUtil.trim(csvRow.get(j));
-                        if ((StrUtil.startWith(cellStr, "[") && StrUtil.endWith(cellStr, "]")) ||
-                                (StrUtil.startWith(cellStr, "{") && StrUtil.endWith(cellStr, "}"))) { // csv文件中json格式双引号会变成两个
-                            cellStr = StrUtil.replace(cellStr, "\"\"", "\"");
-                        }
-						String valName = indexHeader.get(j);
-						Map<String, Object> info = configs.get(valName);
+                CsvReader reader2 = CsvUtil.getReader(ResourceUtil.getReader(tempCsvFile.getPath(), CharsetUtil.CHARSET_UTF_8));
+    		    reader2.setContainsHeader(true);
+    			reader2.stream().forEach(row -> {
+    				// 当前行号
+    				Map<String, String> tmpMap = Tool.MapUtil.newHashMap();
+    				long originalLineNumber = row.getOriginalLineNumber();
+    				boolean ipHandler = false;
+    				Map<String, String> fieldMap = row.getFieldMap();
+    				for(String headerName : fieldMap.keySet()) {
+    					String value = fieldMap.get(headerName);
+    					Map<String, Object> info = configs.get(headerName);
 						String checkType = (String) info.get("checkType");
 						List<String> validateVal = (List<String>) info.get("validateVal");
-						if(StrUtil.equals(valName, "addr_format")) {
+						if(StrUtil.equals(headerName, "addr_format")) {
 							ipHandler = true;
 						}
 						if(StrUtil.equals(checkType, "in")) {
-							if(!validateVal.contains(cellStr)) {
-								log.error("knowledgeBase import data error , line is {},columnName is {}",i,valName);
+							if(!validateVal.contains(value)) {
+								log.error("knowledgeBase import data error , line is {},columnName is {}",originalLineNumber,headerName);
 								throw new CNException(RCode.IMPORT_ERROR);
 							}
 						}else if(StrUtil.equals(checkType, "range")) {
-							if(!(Integer.valueOf(validateVal.get(0))<=Integer.valueOf(cellStr) && Integer.valueOf(validateVal.get(1))>=Integer.valueOf(cellStr))) {
-								log.error("knowledgeBase import data error , line is {},columnName is {}",i,valName);
+							if(!(Integer.valueOf(validateVal.get(0))<=Integer.valueOf(value) && Integer.valueOf(validateVal.get(1))>=Integer.valueOf(value))) {
+								log.error("knowledgeBase import data error , line is {},columnName is {}",originalLineNumber,headerName);
 								throw new CNException(RCode.IMPORT_ERROR);
 							}
 						}else if(StrUtil.equals(checkType, "ip")) {
-							boolean validIpFormat = KnowledgeCheckUtil.validIpFormat(cellStr);
+							boolean validIpFormat = KnowledgeCheckUtil.validIpFormat(value);
 							if(!validIpFormat) {
-								log.error("knowledgeBase import data error , line is {},columnName is {}",i,valName);
+								log.error("knowledgeBase import data error , line is {},columnName is {}",originalLineNumber,headerName);
 								throw new CNException(RCode.IMPORT_ERROR);
 							}
 						}else if(StrUtil.equals(checkType, "notNull")) {
-							if(ObjectUtil.isEmpty(cellStr)) {
-								log.error("knowledgeBase import data error , line is {},columnName is {}",i,valName);
+							if(ObjectUtil.isEmpty(value)) {
+								log.error("knowledgeBase import data error , line is {},columnName is {}",originalLineNumber,headerName);
 								throw new CNException(RCode.IMPORT_ERROR);
 							}
 						}
-                        tmpMap.put(valName, cellStr);
-                    }
-                    // 进行ip1 ip2相关数据内容处理
+                        tmpMap.put(headerName, value);
+    				}
+    				// 进行ip1 ip2相关数据内容处理
                     if(ipHandler) {
                     	String addrFormat = tmpMap.get("addr_format");
                     	String ip1 = tmpMap.get("ip1");
@@ -1164,7 +1170,15 @@ public class KnowledgeBaseServiceImpl extends ServiceImpl<KnowledgeBaseDao, Know
 					if (ObjectUtil.isNotEmpty(tmpMap)) {
 						importDataList.add(tmpMap);
 					}
-				}
+					if(importDataList.size() == 2000) {
+						saveOrUpdateBatch(importDataList,knowledgeBase);
+						importDataList.clear();
+					}
+    			});     
+    			if(ObjectUtil.isNotEmpty(importDataList)) {
+    				saveOrUpdateBatch(importDataList,knowledgeBase);
+    				importDataList.clear();
+    			}
 				break;
 			}
             default: {