feat: add vpn knowledgerelease-24.01-rc2

5 files changed, 106 insertions, 28 deletions

diff --git a/platform-base/src/main/java/com/zdjizhi/base/common/CommonInternalConfig.java b/platform-base/src/main/java/com/zdjizhi/base/common/CommonInternalConfig.java
index 6ca4a1d..06a0c97 100644
--- a/platform-base/src/main/java/com/zdjizhi/base/common/CommonInternalConfig.java
+++ b/platform-base/src/main/java/com/zdjizhi/base/common/CommonInternalConfig.java
@@ -55,9 +55,12 @@ public class CommonInternalConfig {
             .defaultValue("cn_app_tag_user_defined");
 
     //AI Tagging
-    public static final ConfigOption<String> PSIPHON_TYPE = ConfigOptions.key("psiphon.type")
+    public static final ConfigOption<String> IP_VPN_TYPE = ConfigOptions.key("ip.vpn.type")
             .stringType()
-            .defaultValue("cn_psiphon3_ip");
+            .defaultValue("cn_vpn_learning_ip");
+    public static final ConfigOption<String> DOMAIN_VPN_TYPE = ConfigOptions.key("domain.vpn.type")
+            .stringType()
+            .defaultValue("cn_vpn_learning_domain");
 
     public static final ConfigOption<String> ETL_TOPIC = ConfigOptions.key("etl.topic")
             .stringType()
diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/knowledge/KnowledgeManager.java b/platform-etl/src/main/java/com/zdjizhi/etl/knowledge/KnowledgeManager.java
index b191723..f692b2a 100644
--- a/platform-etl/src/main/java/com/zdjizhi/etl/knowledge/KnowledgeManager.java
+++ b/platform-etl/src/main/java/com/zdjizhi/etl/knowledge/KnowledgeManager.java
@@ -6,6 +6,8 @@ import com.zdjizhi.base.common.CommonConfig;
 import com.zdjizhi.base.common.CommonInternalConfig;
 import com.zdjizhi.base.utils.FileUtils;
 import com.zdjizhi.etl.utils.*;
+import com.zdjizhi.etl.utils.ai.DomainVpnUtils;
+import com.zdjizhi.etl.utils.ai.IpVpnUtils;
 import com.zdjizhi.etl.utils.csv.HighCsvReader;
 import com.zdjizhi.etl.utils.fqdn.CategoryUtils;
 import com.zdjizhi.etl.utils.fqdn.IcpUtils;
@@ -74,9 +76,10 @@ public class KnowledgeManager {
                 configuration.get(CommonInternalConfig.INTERNAL_IP_TYPE),
                 configuration.get(CommonInternalConfig.APPSKT_TYPE),
                 configuration.get(CommonInternalConfig.FQDN_WHOIS_TYPE),
-                configuration.get(CommonInternalConfig.PSIPHON_TYPE),
                 configuration.get(CommonInternalConfig.IOC_DARKWEB_TYPE),
-                configuration.get(CommonInternalConfig.IOC_MALWARE_TYPE)
+                configuration.get(CommonInternalConfig.IOC_MALWARE_TYPE),
+                configuration.get(CommonInternalConfig.IP_VPN_TYPE),
+                configuration.get(CommonInternalConfig.DOMAIN_VPN_TYPE)
         );
     }
 
@@ -244,14 +247,21 @@ public class KnowledgeManager {
                 HighCsvReader highCsvReader = new HighCsvReader(reader, needColumns);
                 InternalIpUtils.readInternalIpCsv(highCsvReader);
             }
-            if ((configuration.get(CommonInternalConfig.PSIPHON_TYPE)).equals(type)) {
+            if ((configuration.get(CommonInternalConfig.IP_VPN_TYPE)).equals(type)) {
                 List<String> needColumns = new ArrayList<>();
                 needColumns.add("addr_format");
                 needColumns.add("ip1");
                 needColumns.add("ip2");
-                needColumns.add("type");
+                needColumns.add("vpn_service_name");
                 HighCsvReader highCsvReader = new HighCsvReader(reader, needColumns);
-                PsiphonUtils.readPsiphonCsv(highCsvReader);
+                IpVpnUtils.readCsv(highCsvReader);
+            }
+            if ((configuration.get(CommonInternalConfig.DOMAIN_VPN_TYPE)).equals(type)) {
+                List<String> needColumns = new ArrayList<>();
+                needColumns.add("domain");
+                needColumns.add("vpn_service_name");
+                HighCsvReader highCsvReader = new HighCsvReader(reader, needColumns);
+                DomainVpnUtils.readCsv(highCsvReader);
             }
             if ((configuration.get(CommonInternalConfig.IOC_DARKWEB_TYPE)).equals(type)) {
                 List<String> needColumns = new ArrayList<>();
@@ -288,7 +298,8 @@ public class KnowledgeManager {
         updateKnowledge(configuration.get(CommonInternalConfig.INTERNAL_IP_TYPE), "csv", fileUtils.getFileBytes(configuration.get(CommonInternalConfig.INTERNAL_IP_TYPE) + ".csv"));
         updateKnowledge(configuration.get(CommonInternalConfig.APPSKT_TYPE), "csv", fileUtils.getFileBytes(configuration.get(CommonInternalConfig.APPSKT_TYPE) + ".csv"));
         updateKnowledge(configuration.get(CommonInternalConfig.FQDN_WHOIS_TYPE), "csv", fileUtils.getFileBytes(configuration.get(CommonInternalConfig.FQDN_WHOIS_TYPE) + ".csv"));
-        updateKnowledge(configuration.get(CommonInternalConfig.PSIPHON_TYPE), "csv", fileUtils.getFileBytes(configuration.get(CommonInternalConfig.PSIPHON_TYPE) + ".csv"));
+        updateKnowledge(configuration.get(CommonInternalConfig.IP_VPN_TYPE), "csv", fileUtils.getFileBytes(configuration.get(CommonInternalConfig.IP_VPN_TYPE) + ".csv"));
+        updateKnowledge(configuration.get(CommonInternalConfig.DOMAIN_VPN_TYPE), "csv", fileUtils.getFileBytes(configuration.get(CommonInternalConfig.DOMAIN_VPN_TYPE) + ".csv"));
         updateKnowledge(configuration.get(CommonInternalConfig.IOC_DARKWEB_TYPE), "csv", fileUtils.getFileBytes(configuration.get(CommonInternalConfig.IOC_DARKWEB_TYPE) + ".csv"));
         updateKnowledge(configuration.get(CommonInternalConfig.IOC_MALWARE_TYPE), "csv", fileUtils.getFileBytes(configuration.get(CommonInternalConfig.IOC_MALWARE_TYPE) + ".csv"));
     }
diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/operator/EtlProcessFunc.java b/platform-etl/src/main/java/com/zdjizhi/etl/operator/EtlProcessFunc.java
index e5b2481..27a902d 100644
--- a/platform-etl/src/main/java/com/zdjizhi/etl/operator/EtlProcessFunc.java
+++ b/platform-etl/src/main/java/com/zdjizhi/etl/operator/EtlProcessFunc.java
@@ -6,6 +6,8 @@ import com.zdjizhi.etl.common.CommonConfig;
 import com.zdjizhi.etl.knowledge.MetadataListenerUtils;
 import com.zdjizhi.etl.rule.RuleUpdateListener;
 import com.zdjizhi.etl.utils.*;
+import com.zdjizhi.etl.utils.ai.DomainVpnUtils;
+import com.zdjizhi.etl.utils.ai.IpVpnUtils;
 import com.zdjizhi.etl.utils.fqdn.CategoryUtils;
 import com.zdjizhi.etl.utils.fqdn.IcpUtils;
 import com.zdjizhi.etl.utils.fqdn.WhoisUtils;
@@ -60,7 +62,8 @@ public class EtlProcessFunc extends ProcessFunction<String, CnRecordLog> {
             DnsServerUtils.setWebSktMessage(recordLog);
             AppUtils.setAppSketchMessage(recordLog);
             InternalIpUtils.setInternalIpFlag(recordLog);
-            PsiphonUtils.setPsiphonInfo(recordLog);
+            IpVpnUtils.setInfo(recordLog);
+            DomainVpnUtils.setInfo(recordLog);
             IocDarkWebUtils.setIocDarkWebInfo(recordLog);
             IocMalwareUtils.setIocMalwareInfo(recordLog);
             CustomKnowledgeManager.setTag(recordLog);
diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/ai/DomainVpnUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/ai/DomainVpnUtils.java
new file mode 100644
index 0000000..eca5958
--- /dev/null
+++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/ai/DomainVpnUtils.java
@@ -0,0 +1,68 @@
+package com.zdjizhi.etl.utils.ai;
+
+import com.zdjizhi.base.common.CnRecordLog;
+import com.zdjizhi.etl.utils.csv.HighCsvReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author gujinkai
+ * @version 1.0
+ * @date 2024/1/15 17:43
+ */
+public class DomainVpnUtils {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DomainVpnUtils.class);
+    private static Map<String, String> map = new HashMap<>();
+
+
+    public static void readCsv(HighCsvReader csvReader) {
+        try {
+            Map<String, String> newMap = new HashMap<>((int) (csvReader.getLineNumber() / 0.75F + 1.0F));
+            HighCsvReader.CsvIterator iterator = csvReader.getIterator();
+            while (iterator.hasNext()) {
+                Map<String, String> line = iterator.next();
+                try {
+                    String domain = line.get("domain");
+                    String vpnServiceName = line.get("vpn_service_name");
+
+                    newMap.put(domain, vpnServiceName);
+                } catch (Exception lineException) {
+                    LOG.error("IdcRenterUtils line: " + line.toString() + " parse error：" + lineException, lineException);
+                }
+            }
+            map = newMap;
+        } catch (Exception e) {
+            LOG.error(e.getMessage());
+        }
+    }
+
+
+    public static void setInfo(CnRecordLog cnRecordLog) {
+        try {
+            String domain = cnRecordLog.getDomain();
+            cnRecordLog.putDomain_tag(getInfo(domain));
+        } catch (Exception e) {
+            LOG.error("ETL解析错误\nrecordLog:{} \n", cnRecordLog, e);
+        }
+    }
+
+    private static String getInfo(String domain) {
+        if (domain == null || domain.length() == 0) {
+            return null;
+        }
+        if (map.containsKey(domain)) {
+            return map.get(domain);
+        } else {
+            int index = domain.indexOf(".") + 1;
+            if (index > 0) {
+                return getInfo(domain.substring(index));
+            } else {
+                return null;
+            }
+        }
+    }
+}
diff --git a/platform-etl/src/main/java/com/zdjizhi/etl/utils/PsiphonUtils.java b/platform-etl/src/main/java/com/zdjizhi/etl/utils/ai/IpVpnUtils.java
index 41921ea..51d14bf 100644
--- a/platform-etl/src/main/java/com/zdjizhi/etl/utils/PsiphonUtils.java
+++ b/platform-etl/src/main/java/com/zdjizhi/etl/utils/ai/IpVpnUtils.java
@@ -1,4 +1,4 @@
-package com.zdjizhi.etl.utils;
+package com.zdjizhi.etl.utils.ai;
 
 import com.zdjizhi.base.common.CnRecordLog;
 import com.zdjizhi.etl.utils.common.IPAddress;
@@ -11,13 +11,13 @@ import org.slf4j.LoggerFactory;
 
 import java.util.Map;
 
-public class PsiphonUtils {
+public class IpVpnUtils {
 
-    private static final Logger LOG = LoggerFactory.getLogger(PsiphonUtils.class);
+    private static final Logger LOG = LoggerFactory.getLogger(IpVpnUtils.class);
     private static TreeRangeMap<IPAddress, String> treeRangeMap = TreeRangeMap.create();
 
 
-    public static void readPsiphonCsv(HighCsvReader csvReader) {
+    public static void readCsv(HighCsvReader csvReader) {
         try {
             TreeRangeMap<IPAddress, String> newTreeRangeMap = TreeRangeMap.create();
             HighCsvReader.CsvIterator iterator = csvReader.getIterator();
@@ -27,21 +27,14 @@ public class PsiphonUtils {
                     String addrFormat = line.get("addr_format");
                     String ip1 = line.get("ip1");
                     String ip2 = line.get("ip2");
-                    String type = line.get("type");
-                    if ("0".equals(type)) {
-                        type = "Psiphon3 Server";
-                    } else if ("1".equals(type)) {
-                        type = "Psiphon3 CDN";
-                    } else {
-                        continue;
-                    }
+                    String vpnServiceName = line.get("vpn_service_name");
 
                     if ("Single".equals(addrFormat)) {
                         IPAddress ipAddress = new IPAddress(ip1);
                         if (ipAddress.getIpAddress() == null) {
                             continue;
                         }
-                        newTreeRangeMap.put(Range.closed(ipAddress, ipAddress), type);
+                        newTreeRangeMap.put(Range.closed(ipAddress, ipAddress), vpnServiceName);
                     }
                     if ("Range".equals(addrFormat)) {
                         IPAddress startIpAddress = new IPAddress(ip1);
@@ -49,7 +42,7 @@ public class PsiphonUtils {
                         if (startIpAddress.getIpAddress() == null || endIpAddress.getIpAddress() == null) {
                             continue;
                         }
-                        newTreeRangeMap.put(Range.closed(startIpAddress, endIpAddress), type);
+                        newTreeRangeMap.put(Range.closed(startIpAddress, endIpAddress), vpnServiceName);
                     }
                     if ("CIDR".equals(addrFormat)) {
                         inet.ipaddr.IPAddress cidrIpAddress = new IPAddressString(ip1 + "/" + ip2).getAddress();
@@ -58,7 +51,7 @@ public class PsiphonUtils {
                         }
                         inet.ipaddr.IPAddress startIpAddress = cidrIpAddress.getLower();
                         inet.ipaddr.IPAddress endIpAddress = cidrIpAddress.getUpper();
-                        newTreeRangeMap.put(Range.closed(new IPAddress(startIpAddress), new IPAddress(endIpAddress)), type);
+                        newTreeRangeMap.put(Range.closed(new IPAddress(startIpAddress), new IPAddress(endIpAddress)), vpnServiceName);
                     }
                 } catch (Exception lineException) {
                     LOG.error("IdcRenterUtils line: " + line.toString() + " parse error：" + lineException, lineException);
@@ -71,13 +64,13 @@ public class PsiphonUtils {
     }
 
 
-    public static void setPsiphonInfo(CnRecordLog cnRecordLog) {
+    public static void setInfo(CnRecordLog cnRecordLog) {
         try {
             IPAddress ipAddress = new IPAddress(cnRecordLog.getCommon_server_ip());
             if (ipAddress.getIpAddress() != null) {
-                String type = treeRangeMap.get(ipAddress);
-                if (type != null) {
-                    cnRecordLog.putServer_ip_tag(type);
+                String vpnServiceName = treeRangeMap.get(ipAddress);
+                if (vpnServiceName != null) {
+                    cnRecordLog.putServer_ip_tag(vpnServiceName);
                 }
             }
         } catch (Exception e) {