diff options
4 files changed, 28 insertions, 25 deletions
diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/DnsServerUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/DnsServerUtils.java index ccac029..6a184c0 100644 --- a/platform-etl/src/main/java/com/zdjizhi/etl/utils/DnsServerUtils.java +++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/DnsServerUtils.java @@ -38,8 +38,9 @@ public class DnsServerUtils { try { CSVReader csvReader = new CSVReader(serverReader, CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1); - dnsmap = new HashMap<>(csvReader.readAll().size() * 2); - for (String[] strs : csvReader) { + List<String[]> data = csvReader.readAll(); + dnsmap = new HashMap<>(data.size() * 2); + for (String[] strs : data) { String key = strs[1]; if (!key.startsWith("#")) { DnsServerInfo catalog = new DnsServerInfo(); diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/FcUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/FcUtils.java index 92fe941..8472465 100644 --- a/platform-etl/src/main/java/com/zdjizhi/etl/utils/FcUtils.java +++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/FcUtils.java @@ -4,6 +4,7 @@ import com.opencsv.CSVParser; import com.opencsv.CSVReader; import com.zdjizhi.base.common.CnRecordLog; import com.zdjizhi.base.common.CommonConfig; +import com.zdjizhi.base.utils.FileByteUtils; import com.zdjizhi.etl.common.DomainCategoryReputation; import com.zdjizhi.etl.common.DomainIcp; import org.slf4j.Logger; @@ -12,6 +13,7 @@ import org.slf4j.LoggerFactory; import java.io.*; import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.List; import java.util.Set; public class FcUtils { @@ -24,8 +26,9 @@ public class FcUtils { // DataInputStream in = new DataInputStream(new FileInputStream(new File(path))); CSVReader csvReader = new CSVReader(websktReader, CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1); - smap = new HashMap<>(csvReader.readAll().size() * 2); - for (String[] strs : csvReader) { + List<String[]> data = csvReader.readAll(); + smap = new HashMap<>(data.size() * 2); + for (String[] strs : data) { String key = strs[0]; if (!key.startsWith("#")){ DomainCategoryReputation catalog = new DomainCategoryReputation(); @@ -50,8 +53,9 @@ public class FcUtils { // DataInputStream in = new DataInputStream(new FileInputStream(new File(path))); CSVReader csvReader = new CSVReader(icpReader, CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1); - icpmap = new HashMap<>(csvReader.readAll().size() * 2); - for (String[] strs : csvReader) { + List<String[]> data = csvReader.readAll(); + icpmap = new HashMap<>(data.size() * 2); + for (String[] strs : data) { String key = strs[0]; if (!key.startsWith("#")){ DomainIcp catalog = new DomainIcp(); @@ -95,19 +99,13 @@ public class FcUtils { } -// public static void main(String[] args) throws Exception { -//// HashMap<String, DomainCategoryReputation> map = readCsv(CommonConfig.WEBSKT_PATH); -//// HashMap<String, CnRecordLog> map = readIcpCsv(CommonConfig.ICP_PATH); -// Set<String> keySet = smap.keySet(); -// for (String key : keySet){ -// System.out.println(smap.get(key).toString()); -// } -// System.out.println(smap.size()); -// -// Set<String> keySet1 = icpmap.keySet(); -// for (String key : keySet){ -// System.out.println(icpmap.get(key).toString()); -// } -// System.out.println(icpmap.size()); -// } + public static void main(String[] args) throws IOException { + byte[] websktBytes = FileByteUtils.getFileBytes( "C:\\\\Users\\\\admin\\\\Desktop\\\\tmp\\\\olddata\\\\test.csv"); + if (websktBytes != null) { + ByteArrayInputStream websktInputStream = new ByteArrayInputStream(websktBytes); + InputStreamReader reader = new InputStreamReader(websktInputStream); + readCsv(reader); + } + + } }
\ No newline at end of file diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/IspUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/IspUtils.java index cd61672..39339dc 100644 --- a/platform-etl/src/main/java/com/zdjizhi/etl/utils/IspUtils.java +++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/IspUtils.java @@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory; import java.io.*; import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.List; import java.util.Set; /** @@ -27,8 +28,9 @@ public class IspUtils { // DataInputStream in = new DataInputStream(new FileInputStream(new File(path))); CSVReader csvReader = new CSVReader(ispReader, CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1); - ispmap = new HashMap<>(csvReader.readAll().size() * 2); - for (String[] strs : csvReader) { + List<String[]> data = csvReader.readAll(); + ispmap = new HashMap<>(data.size() * 2); + for (String[] strs : data) { String key = strs[1]; LinkAndIspInfo linkDirectionInfo = new LinkAndIspInfo(); linkDirectionInfo.setRegion(strs[3]); diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/LinkUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/LinkUtils.java index 6f43d0a..6443fc7 100644 --- a/platform-etl/src/main/java/com/zdjizhi/etl/utils/LinkUtils.java +++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/LinkUtils.java @@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory; import java.io.*; import java.util.HashMap; +import java.util.List; import java.util.Set; /** @@ -25,8 +26,9 @@ public class LinkUtils { try { CSVReader csvReader = new CSVReader(linkReader, CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER, 1); - linkmap = new HashMap<>(csvReader.readAll().size() * 2); - for (String[] strs : csvReader) { + List<String[]> data = csvReader.readAll(); + linkmap = new HashMap<>(data.size() * 2); + for (String[] strs : data) { long key = Long.parseLong(strs[6]); LinkAndIspInfo linkDirectionInfo = new LinkAndIspInfo(); linkDirectionInfo.setEgress_link_direction(strs[4]); |