Modified: cyberghost 新增udp探测行为获取IP逻辑

3 files changed, 58 insertions, 16 deletions

diff --git a/config23.10.yaml b/config23.10.yaml
index 0e217d6..6921ba0 100644
--- a/config23.10.yaml
+++ b/config23.10.yaml
@@ -7,7 +7,7 @@ common:
     active_scan: # max calls/s (rough estimate) = max workers * max_call_per_sec
         switch: on
         max_workers: 10
-        max_calls_per_sec: 10
+        max_calls_per_sec: 1000
 
 monitor:
     monitor_file_path: /opt/vpn-thwarting/vpn_knolwdgebase_monitor.prom
@@ -104,6 +104,8 @@ cyberghostvpn:
     ip:
         object_type: ip
         kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'cyberghostvpn'
+        udp_monitor_app_name: Cyberghost-UDP
+        sql: SELECT DISTINCT common_server_ip FROM {$db_name}.{$table_name} WHERE {$time_filter} and application_full_path like '%{$udp_monitor_app_name}%'
 
 
 windscribevpn:
diff --git a/config24.01.yaml b/config24.01.yaml
index 31e9664..784461b 100644
--- a/config24.01.yaml
+++ b/config24.01.yaml
@@ -5,8 +5,8 @@ common:
     time_filter_pattern: (recv_time_columnname> toDateTime('{$start_time}', '{$time_zone}')) AND(recv_time_columnname <= toDateTime('{$end_time}', '{$time_zone}'))
     save_knowledgebase: True
     active_scan:  # max calls/s (rough estimate) = max workers * max_call_per_sec
-        switch: off
-        max_workers: 10
+        switch: on
+        max_workers: 100
         max_calls_per_sec: 10
 
 monitor:
@@ -93,10 +93,10 @@ protonvpn:
     sql: SELECT server_ip, groupUniqArray(server_port) AS ports FROM {$db_name}.{$table_name} WHERE {$time_filter} AND (server_port IN (443, 7770, 8443, 88, 5060, 51820, 500, 80, 1224, 4500, 4569, 5060, 1194)) GROUP BY server_ip HAVING length(ports) > 10
 
 
-
+# tsg系统内置Cyberghost-UDP APP获取新增活跃IP
 cyberghostvpn:
-    plugin_name: cyberghost
-    vpn_service_name: cyberghost
+    plugin_name: cyberghostvpn
+    vpn_service_name: cyberghostvpn
     plugin_id: 5
     confidence: confirmed
     domain:
@@ -105,6 +105,8 @@ cyberghostvpn:
     ip:
         object_type: ip
         kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'cyberghostvpn'
+        udp_monitor_app_name: Cyberghost-UDP
+        sql: SELECT DISTINCT server_ip FROM {$db_name}.{$table_name} WHERE {$time_filter} and app_transition like '%{$udp_monitor_app_name}%'
 
 
 windscribevpn:
@@ -120,7 +122,6 @@ windscribevpn:
         object_type: ip
         kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'windscribevpn'
 
-
 turbovpn:
     vpn_service_name: turbovpn
     plugin_id: 7
diff --git a/detection/vpnservices/cyberghostvpn.py b/detection/vpnservices/cyberghostvpn.py
index 9cd2eae..9359529 100644
--- a/detection/vpnservices/cyberghostvpn.py
+++ b/detection/vpnservices/cyberghostvpn.py
@@ -43,7 +43,7 @@ class Cyberghostvpn(VpnDetector):
         result_group.extend(cyberghostvpn_detector.find_server())
 
         # start finding cyberghostvpn server ip
-        cyberghostvpn_detector = CyberghostvpnServerip()
+        cyberghostvpn_detector = CyberghostvpnServerip(self.start_time, self.end_time)
         result_group.extend(cyberghostvpn_detector.find_server())
 
         return result_group
@@ -55,13 +55,14 @@ class CyberghostvpnServerip(VpnDetector):
     This class is used to detect cyberghostvpn server ip
     """
 
-    def __init__(self):
-        super().__init__('', '')
+    def __init__(self, start_time, end_time):
+        super().__init__(start_time, end_time)
         self.plugin_config = self.load_config()['cyberghostvpn']
         self.plugin_name = self.plugin_config['plugin_name']
         self.object_type = self.plugin_config['ip']['object_type']
         # 开始时间为当前的整点时间
-        self.start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:00:00")
+        self.start_time = start_time
+        self.end_time = end_time
         self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ',
                                                                                                                '_').replace(
             ':', '')[:13] + '.csv'
@@ -77,6 +78,8 @@ class CyberghostvpnServerip(VpnDetector):
         self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name']
         self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name']
 
+        self.sql = self.plugin_config['ip']['sql']
+
 
     def find_more_servernames(self, server_name_list):
         """
@@ -88,12 +91,15 @@ class CyberghostvpnServerip(VpnDetector):
 
 
         for server_name in server_name_list:
-            pattern = re.compile(r'\.(.*?)\-rack')
-            pattern_list.append(pattern.findall(server_name)[0])
+            # pattern = re.compile(r'\.(.*?)\-rack')
+            pattern = re.compile(r'\.(.*?)\.nodes')
+            findall = pattern.findall(server_name)
+            if len(findall) > 0:
+                pattern_list.append(findall[0])
         pattern_list = set(pattern_list)
 
         for pattern_str in pattern_list:
-            domain_list = [f"blade{str(index1)}.{pattern_str}-rack4{str(index2).zfill(2)}.nodes.gen4.ninja" for index1 in range(1, 100) for index2 in range(1, 100)]
+            domain_list = [f"blade{str(index1)}.{pattern_str}.nodes.gen4.ninja" for index1 in range(1, 100)]
             expanded_server_names.extend(domain_list)
 
         return expanded_server_names
@@ -107,6 +113,8 @@ class CyberghostvpnServerip(VpnDetector):
         self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace(
             "{$mariadb_domain_tablename}", self.mariadb_domain_tb_name)
 
+        # 根据server name获取ip
+        self.logger.info('[{}] - Get servername from knowledge base.'.format(self.plugin_name))
         servername_list = []
         resolved_ip_list = []
         try:
@@ -118,7 +126,7 @@ class CyberghostvpnServerip(VpnDetector):
             servername_list = [i[0] for i in query_result]
 
         # 判断是否能够访问外网，如果能够访问外网，则从外网获取cyberghost_servername_list的域名解析地址
-        if self.config['common']['active_scan']['switch']=='on' and check_internet():
+        if self.config['common']['active_scan']['switch'] and check_internet():
             servername_list = self.find_more_servernames(servername_list)
             if len(servername_list) > 0:
                 resolved_ip_list = self.resolve_dns_for_domain_list(servername_list)
@@ -129,7 +137,38 @@ class CyberghostvpnServerip(VpnDetector):
         else:
             self.logger.info('[{}] - No internet connection, skip dns resolve.'.format(self.plugin_name))
 
-        return [ServerGroup(self.object_type, resolved_ip_list, self.output_file_name)]
+
+        # 根据udp探测行为获取IP
+        self.logger.info('[{}] - Start to query server ip from session records.'.format(self.plugin_name))
+
+        TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname',
+                                                                                   self.config['common'][
+                                                                                       'recv_time_columnname'])
+        time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str(
+            self.end_time)).replace("{$time_zone}", self.time_zone)
+        self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name)
+        self.sql = self.sql.replace("{$time_filter}", time_filter)
+        self.sql = self.sql.replace("{$udp_monitor_app_name}", self.plugin_config['ip']['udp_monitor_app_name'])
+
+        self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_name, self.sql))
+
+        # query data from clickhouse database
+        try:
+            cyberghostvpn_serverip_df = pd.DataFrame(self.client.execute(self.sql))
+        finally:
+            self.client.disconnect()
+
+        if cyberghostvpn_serverip_df.empty:
+            self.logger.info('[{}] - No server ip found from session records'.format(self.plugin_name))
+            return []
+        cyberghostvpn_serverip_list = cyberghostvpn_serverip_df[0].drop_duplicates().tolist()
+        self.logger.info('[{}] - Query server ip from clickhouse database successfully. {} items found'
+                         .format(self.plugin_name, len(cyberghostvpn_serverip_list)))
+
+
+        result_ip_list = list(set(cyberghostvpn_serverip_list + resolved_ip_list))
+
+        return [ServerGroup(self.object_type, result_ip_list, self.output_file_name)]