修复部分知识库未加载入内存的问题release-23.02-rc2

4 files changed, 28 insertions, 25 deletions

diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/DnsServerUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/DnsServerUtils.java
index ccac029..6a184c0 100644
--- a/platform-etl/src/main/java/com/zdjizhi/etl/utils/DnsServerUtils.java
+++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/DnsServerUtils.java
@@ -38,8 +38,9 @@ public class DnsServerUtils {
         try {
             CSVReader csvReader = new CSVReader(serverReader, CSVParser.DEFAULT_SEPARATOR,
                     CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1);
-            dnsmap = new HashMap<>(csvReader.readAll().size() * 2);
-            for (String[] strs : csvReader) {
+            List<String[]> data = csvReader.readAll();
+            dnsmap = new HashMap<>(data.size() * 2);
+            for (String[] strs : data) {
                 String key = strs[1];
                 if (!key.startsWith("#")) {
                     DnsServerInfo catalog = new DnsServerInfo();
diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/FcUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/FcUtils.java
index 92fe941..8472465 100644
--- a/platform-etl/src/main/java/com/zdjizhi/etl/utils/FcUtils.java
+++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/FcUtils.java
@@ -4,6 +4,7 @@ import com.opencsv.CSVParser;
 import com.opencsv.CSVReader;
 import com.zdjizhi.base.common.CnRecordLog;
 import com.zdjizhi.base.common.CommonConfig;
+import com.zdjizhi.base.utils.FileByteUtils;
 import com.zdjizhi.etl.common.DomainCategoryReputation;
 import com.zdjizhi.etl.common.DomainIcp;
 import org.slf4j.Logger;
@@ -12,6 +13,7 @@ import org.slf4j.LoggerFactory;
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Set;
 
 public class FcUtils {
@@ -24,8 +26,9 @@ public class FcUtils {
 //            DataInputStream in = new DataInputStream(new FileInputStream(new File(path)));
             CSVReader csvReader = new CSVReader(websktReader, CSVParser.DEFAULT_SEPARATOR,
                     CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1);
-            smap = new HashMap<>(csvReader.readAll().size() * 2);
-            for (String[] strs : csvReader) {
+            List<String[]> data = csvReader.readAll();
+            smap = new HashMap<>(data.size() * 2);
+            for (String[] strs : data) {
                 String key = strs[0];
                 if (!key.startsWith("#")){
                     DomainCategoryReputation catalog = new DomainCategoryReputation();
@@ -50,8 +53,9 @@ public class FcUtils {
 //            DataInputStream in = new DataInputStream(new FileInputStream(new File(path)));
             CSVReader csvReader = new CSVReader(icpReader, CSVParser.DEFAULT_SEPARATOR,
                     CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1);
-            icpmap = new HashMap<>(csvReader.readAll().size() * 2);
-            for (String[] strs : csvReader) {
+            List<String[]> data = csvReader.readAll();
+            icpmap = new HashMap<>(data.size() * 2);
+            for (String[] strs : data) {
                 String key = strs[0];
                 if (!key.startsWith("#")){
                     DomainIcp catalog = new DomainIcp();
@@ -95,19 +99,13 @@ public class FcUtils {
     }
 
 
-//    public static void main(String[] args) throws Exception {
-////        HashMap<String, DomainCategoryReputation> map = readCsv(CommonConfig.WEBSKT_PATH);
-////        HashMap<String, CnRecordLog> map = readIcpCsv(CommonConfig.ICP_PATH);
-//        Set<String> keySet = smap.keySet();
-//        for (String key : keySet){
-//            System.out.println(smap.get(key).toString());
-//        }
-//        System.out.println(smap.size());
-//
-//        Set<String> keySet1 = icpmap.keySet();
-//        for (String key : keySet){
-//            System.out.println(icpmap.get(key).toString());
-//        }
-//        System.out.println(icpmap.size());
-//    }
+    public static void main(String[] args) throws IOException {
+        byte[] websktBytes = FileByteUtils.getFileBytes( "C:\\\\Users\\\\admin\\\\Desktop\\\\tmp\\\\olddata\\\\test.csv");
+        if (websktBytes != null) {
+            ByteArrayInputStream websktInputStream = new ByteArrayInputStream(websktBytes);
+            InputStreamReader reader = new InputStreamReader(websktInputStream);
+            readCsv(reader);
+        }
+
+    }
 }
\ No newline at end of file
diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/IspUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/IspUtils.java
index cd61672..39339dc 100644
--- a/platform-etl/src/main/java/com/zdjizhi/etl/utils/IspUtils.java
+++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/IspUtils.java
@@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory;
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Set;
 
 /**
@@ -27,8 +28,9 @@ public class IspUtils {
 //            DataInputStream in = new DataInputStream(new FileInputStream(new File(path)));
             CSVReader csvReader = new CSVReader(ispReader, CSVParser.DEFAULT_SEPARATOR,
                     CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1);
-            ispmap = new HashMap<>(csvReader.readAll().size() * 2);
-            for (String[] strs : csvReader) {
+            List<String[]> data = csvReader.readAll();
+            ispmap = new HashMap<>(data.size() * 2);
+            for (String[] strs : data) {
                 String key = strs[1];
                 LinkAndIspInfo linkDirectionInfo = new LinkAndIspInfo();
                 linkDirectionInfo.setRegion(strs[3]);
diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/LinkUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/LinkUtils.java
index 6f43d0a..6443fc7 100644
--- a/platform-etl/src/main/java/com/zdjizhi/etl/utils/LinkUtils.java
+++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/LinkUtils.java
@@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.*;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Set;
 
 /**
@@ -25,8 +26,9 @@ public class LinkUtils {
         try {
             CSVReader csvReader = new CSVReader(linkReader, CSVParser.DEFAULT_SEPARATOR,
                     CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1);
-            linkmap = new HashMap<>(csvReader.readAll().size() * 2);
-            for (String[] strs : csvReader) {
+            List<String[]> data = csvReader.readAll();
+            linkmap = new HashMap<>(data.size() * 2);
+            for (String[] strs : data) {
                 long key = Long.parseLong(strs[6]);
                 LinkAndIspInfo linkDirectionInfo = new LinkAndIspInfo();
                 linkDirectionInfo.setEgress_link_direction(strs[4]);