v24.01

7 files changed, 126 insertions, 51 deletions

diff --git a/detection/knowledgebase_monitor.py b/detection/knowledgebase_monitor.py
index 588cedd..ae69797 100644
--- a/detection/knowledgebase_monitor.py
+++ b/detection/knowledgebase_monitor.py
@@ -64,7 +64,7 @@ class KnowledgeBaseMonitor:
             q += " and {} < '{}'".format(time_column, end_t)
 
         q = q.lstrip(' and')
-        print(vpn_service, q)
+        # print(vpn_service, q)
         count_num = self.knowledgebase_tool.get_knowledgebase_count(knowledge_id=library_id, q=q)
         return count_num
 
@@ -76,13 +76,13 @@ class KnowledgeBaseMonitor:
         if vpn_service is None: vpn_service='all'
 
         # cycle active
-        kb_metric['{}_active_ip_count'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap,
+        kb_metric['active_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap,
                                                                 start_t=start_time, end_t=end_time, vpn_service=vpn_service)
-        kb_metric['{}_new_ip_count'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='new', timezone_gap_hour=time_zone_gap,
+        kb_metric['new_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='new', timezone_gap_hour=time_zone_gap,
                                                                               start_t=start_time, end_t=end_time, vpn_service=vpn_service)
-        kb_metric['{}_active_domain_count'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap,
+        kb_metric['active_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap,
                                                                 start_t=start_time, end_t=end_time, vpn_service=vpn_service)
-        kb_metric['{}_new_domain_count'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='new', timezone_gap_hour=time_zone_gap,
+        kb_metric['new_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='new', timezone_gap_hour=time_zone_gap,
                                                              start_t=start_time, end_t=end_time, vpn_service=vpn_service)
 
 
@@ -109,32 +109,32 @@ class KnowledgeBaseMonitor:
         if vpn_service is None: vpn_service = 'all'
 
         # all
-        kb_metric['{}_ip_count'.format(vpn_service)] = self.get_vpn_count(node_type='ip', timezone_gap_hour=time_zone_gap, vpn_service=vpn_service)
-        kb_metric['{}_domain_count'.format(vpn_service)] = self.get_vpn_count(node_type='domain', timezone_gap_hour=time_zone_gap, vpn_service=vpn_service)
+        kb_metric['ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', timezone_gap_hour=time_zone_gap, vpn_service=vpn_service)
+        kb_metric['domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', timezone_gap_hour=time_zone_gap, vpn_service=vpn_service)
 
         # all active in like 7 days
         t = (datetime.datetime.now().replace(minute=0, second=0, microsecond=0)
              - datetime.timedelta(days=self.monitor_config['outdated_days'])).strftime("%Y-%m-%d %H:%M:%S")
-        kb_metric['{}_active_ip_count'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap, start_t=t, vpn_service=vpn_service)
-        kb_metric['{}_active_domain_count'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap, start_t=t, vpn_service=vpn_service)
+        kb_metric['active_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap, start_t=t, vpn_service=vpn_service)
+        kb_metric['active_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap, start_t=t, vpn_service=vpn_service)
 
 
         # cycle active
-        kb_metric['{}_cycle_active_ip_count'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap,
+        kb_metric['cycle_active_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap,
                                                                 start_t=current_start_time, end_t=current_end_time, vpn_service=vpn_service)
-        kb_metric['{}_cycle_new_ip_count'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='new', timezone_gap_hour=time_zone_gap,
+        kb_metric['cycle_new_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='new', timezone_gap_hour=time_zone_gap,
                                                                 start_t=current_start_time, end_t=current_end_time, vpn_service=vpn_service)
-        kb_metric['{}_cycle_active_domain_count'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap,
+        kb_metric['cycle_active_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap,
                                                                 start_t=current_start_time, end_t=current_end_time, vpn_service=vpn_service)
-        kb_metric['{}_cycle_new_domain_count'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='new', timezone_gap_hour=time_zone_gap,
+        kb_metric['cycle_new_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='new', timezone_gap_hour=time_zone_gap,
                                                              start_t=current_start_time, end_t=current_end_time, vpn_service=vpn_service)
 
 
         # churn ratio = (# new in current cycle)/(# all active)
-        kb_metric['{}_cycle_ip_churn_ratio'.format(vpn_service)] = np.round(kb_metric['{}_cycle_new_ip_count'.format(vpn_service)]/kb_metric['{}_active_ip_count'.format(vpn_service)], 4) \
-            if kb_metric['{}_active_ip_count'.format(vpn_service)] > 0 else 0
-        kb_metric['{}_cycle_domain_churn_ratio'.format(vpn_service)] = np.round(kb_metric['{}_cycle_new_domain_count'.format(vpn_service)] / kb_metric['{}_active_domain_count'.format(vpn_service)], 4) \
-            if kb_metric['{}_active_domain_count'.format(vpn_service)] > 0 else 0
+        kb_metric['cycle_ip_churn_ratio{{type="{}"}}'.format(vpn_service)] = np.round(kb_metric['cycle_new_ip_count{{type="{}"}}'.format(vpn_service)]/kb_metric['active_ip_count{{type="{}"}}'.format(vpn_service)], 4) \
+            if kb_metric['active_ip_count{{type="{}"}}'.format(vpn_service)] > 0 else 0
+        kb_metric['cycle_domain_churn_ratio{{type="{}"}}'.format(vpn_service)] = np.round(kb_metric['cycle_new_domain_count{{type="{}"}}'.format(vpn_service)] / kb_metric['active_domain_count{{type="{}"}}'.format(vpn_service)], 4) \
+            if kb_metric['active_domain_count{{type="{}"}}'.format(vpn_service)] > 0 else 0
         return kb_metric
 
 
@@ -172,6 +172,10 @@ if __name__ == '__main__':
             print('Please input correct recent interval')
             exit()
 
+        # 根据当前时间向前取整小时
+        end_time = datetime.datetime.now().strftime("%Y-%m-%d %H:00:00")
+        start_time = (datetime.datetime.now() - datetime.timedelta(hours=recent_interval)).strftime("%Y-%m-%d %H:00:00")
+
         monitor_result_dict.update(monitor.calculate_vpn_monitor_recent_mode(vpn_service=None))
         vpn_service_name_list = []
         for plugin_name in config_dict:
diff --git a/detection/vpn_detector.py b/detection/vpn_detector.py
index 1685748..52e1d91 100644
--- a/detection/vpn_detector.py
+++ b/detection/vpn_detector.py
@@ -132,6 +132,7 @@ class VpnDetector:
         except Exception as e:
             # self.logger.error("Resolve failed. {}: {} ".format(server_name, e))
             pass
+        self.logger.info('{}  {}'.format(server_name, resolved_addr))
 
         return server_name, resolved_addr
 
diff --git a/detection/vpnservices/cyberghostvpn.py b/detection/vpnservices/cyberghostvpn.py
index 9359529..2aae173 100644
--- a/detection/vpnservices/cyberghostvpn.py
+++ b/detection/vpnservices/cyberghostvpn.py
@@ -39,12 +39,14 @@ class Cyberghostvpn(VpnDetector):
         result_group = []
 
         # start finding cyberghostvpn server name
-        cyberghostvpn_detector = CyberghostvpnServername(self.start_time, self.end_time)
-        result_group.extend(cyberghostvpn_detector.find_server())
+        cyberghostvpn_server_name_detector = CyberghostvpnServername(self.start_time, self.end_time)
+        server_name_object = cyberghostvpn_server_name_detector.find_server()
+        result_group.extend(server_name_object)
 
         # start finding cyberghostvpn server ip
-        cyberghostvpn_detector = CyberghostvpnServerip(self.start_time, self.end_time)
-        result_group.extend(cyberghostvpn_detector.find_server())
+        cyberghostvpn_server_ip_detector = CyberghostvpnServerip(self.start_time, self.end_time)
+        cyberghostvpn_server_ip_detector.server_name_list = server_name_object[0].server_list if len(server_name_object) > 0 else []
+        result_group.extend(cyberghostvpn_server_ip_detector.find_server())
 
         return result_group
 
@@ -80,6 +82,8 @@ class CyberghostvpnServerip(VpnDetector):
 
         self.sql = self.plugin_config['ip']['sql']
 
+        self.server_name_list = []
+
 
     def find_more_servernames(self, server_name_list):
         """
@@ -105,6 +109,21 @@ class CyberghostvpnServerip(VpnDetector):
         return expanded_server_names
 
 
+    def find_server_name_patterns(self, server_name_list):
+        pattern_list = []
+
+        for server_name in server_name_list:
+            # pattern = re.compile(r'\.(.*?)\-rack')
+            pattern = re.compile(r'\.(.*?)\.nodes')
+            findall = pattern.findall(server_name)
+            if len(findall) > 0:
+                pattern_list.append(findall[0])
+        pattern_list = set(pattern_list)
+
+        return pattern_list
+
+
+
     def find_server(self):
         """
         Get cyberghostvpn server ip by resolving cyberghostvpn server name
@@ -124,16 +143,31 @@ class CyberghostvpnServerip(VpnDetector):
 
         if query_result:
             servername_list = [i[0] for i in query_result]
+        self.server_name_list.extend(servername_list)
 
         # 判断是否能够访问外网，如果能够访问外网，则从外网获取cyberghost_servername_list的域名解析地址
         if self.config['common']['active_scan']['switch'] and check_internet():
-            servername_list = self.find_more_servernames(servername_list)
-            if len(servername_list) > 0:
-                resolved_ip_list = self.resolve_dns_for_domain_list(servername_list)
+            # servername_list = self.find_more_servernames(servername_list)
+            # if len(servername_list) > 0:
+            #     resolved_ip_list = self.resolve_dns_for_domain_list(servername_list)
+            #     self.logger.info('[{}] - Get {} server ip by resolving server name successfully.'.format(self.plugin_name, len(resolved_ip_list)))
+            # else:
+            #     self.logger.info(
+            #         '[{}] - No cyberghost server name found from knowledge database.'.format(self.plugin_name))
+
+            server_rackname_list = self.find_server_name_patterns(self.server_name_list)
+            if len(server_rackname_list) > 0:
+                resolved_ip_list = []
+                for rack_name in server_rackname_list:
+                    index = 1
+                    resolve_result = self.get_resolved_addr(f"blade{str(index)}.{rack_name}.nodes.gen4.ninja")
+                    while resolve_result[1] is not None:
+                        self.logger.info('{}  {}'.format(resolve_result[0], resolve_result[1]))
+                        resolved_ip_list.extend(resolve_result[1])
+                        index += 1
+                        resolve_result = self.get_resolved_addr(f"blade{str(index)}.{rack_name}.nodes.gen4.ninja")
                 self.logger.info('[{}] - Get {} server ip by resolving server name successfully.'.format(self.plugin_name, len(resolved_ip_list)))
-            else:
-                self.logger.info(
-                    '[{}] - No cyberghost server name found from knowledge database.'.format(self.plugin_name))
+
         else:
             self.logger.info('[{}] - No internet connection, skip dns resolve.'.format(self.plugin_name))
 
diff --git a/detection/vpnservices/ipvanishvpn.py b/detection/vpnservices/ipvanishvpn.py
index 1ff908c..1b1d5fb 100644
--- a/detection/vpnservices/ipvanishvpn.py
+++ b/detection/vpnservices/ipvanishvpn.py
@@ -40,12 +40,16 @@ class Ipvanishvpn(VpnDetector):
         result_group = []
 
         # start finding ipvanishvpn server name
-        ipvanishvpn_detector = IpvanishvpnServername(self.start_time, self.end_time)
-        result_group.extend(ipvanishvpn_detector.find_server())
+        ipvanishvpn_server_name_detector = IpvanishvpnServername(self.start_time, self.end_time)
+        server_name_object = ipvanishvpn_server_name_detector.find_server()
+        result_group.extend(server_name_object)
 
         # start finding ipvanishvpn server ip
-        ipvanishvpn_detector = IpvanishvpnServerip()
-        result_group.extend(ipvanishvpn_detector.find_server())
+        ipvanishvpn_server_ip_detector = IpvanishvpnServerip()
+        # server_name_list 初始化本周期已查询到的
+        ipvanishvpn_server_ip_detector.server_name_list = server_name_object[0].server_list if len(
+            server_name_object) > 0 else []
+        result_group.extend(ipvanishvpn_server_ip_detector.find_server())
 
         return result_group
 
@@ -76,6 +80,8 @@ class IpvanishvpnServerip(VpnDetector):
         self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name']
         self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name']
 
+        self.server_name_list = []
+
 
     def find_more_servernames(self, server_name_list):
         """
@@ -116,11 +122,18 @@ class IpvanishvpnServerip(VpnDetector):
         if query_result:
             servername_list = [i[0] for i in query_result]
 
+        self.server_name_list.extend(servername_list)
+
         # 判断是否能够访问外网，如果能够访问外网，则从外网获取ipvanish_servername_list的域名解析地址
         if self.config['common']['active_scan']['switch'] and check_internet():
             servername_list = self.find_more_servernames(servername_list)
             if len(servername_list) > 0:
-                resolved_ip_list = self.resolve_dns_for_domain_list(servername_list)
+                resolved_ip_list = self.resolve_dns_for_domain_list(self.server_name_list,
+                                                                  max_workers=self.config['common']['active_scan'][
+                                                                      'max_workers'],
+                                                                  max_calls_per_second=
+                                                                  self.config['common']['active_scan'][
+                                                                      'max_calls_per_sec'])
                 self.logger.info(
                     '[{}] - Get {} server ip by resolving server name successfully.'.format(
                         self.plugin_name, len(resolved_ip_list)))
diff --git a/detection/vpnservices/ivacyvpn.py b/detection/vpnservices/ivacyvpn.py
index 918e14e..e12fd9c 100644
--- a/detection/vpnservices/ivacyvpn.py
+++ b/detection/vpnservices/ivacyvpn.py
@@ -37,12 +37,15 @@ class Ivacyvpn(VpnDetector):
         result_group = []
 
         # start finding ivacyvpn server name
-        ivacyvpn_detector = IvacyvpnServername(self.start_time, self.end_time)
-        result_group.extend(ivacyvpn_detector.find_server())
+        ivacyvpn_server_name_detector = IvacyvpnServername(self.start_time, self.end_time)
+        server_name_object = ivacyvpn_server_name_detector.find_server()
+        result_group.extend(server_name_object)
 
         # start finding ivacyvpn server ip
-        ivacyvpn_detector = IvacyvpnServerip()
-        result_group.extend(ivacyvpn_detector.find_server())
+        ivacyvpn_server_ip_detector = IvacyvpnServerip()
+        # server_name_list 初始化本周期已查询到的
+        ivacyvpn_server_ip_detector.server_name_list = server_name_object[0].server_list if len(server_name_object) > 0 else []
+        result_group.extend(ivacyvpn_server_ip_detector.find_server())
 
         return result_group
 
@@ -74,6 +77,8 @@ class IvacyvpnServerip(VpnDetector):
         self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name']
         self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name']
 
+        self.server_name_list = []
+
 
 
 
@@ -84,7 +89,6 @@ class IvacyvpnServerip(VpnDetector):
         """
         self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_domain_tablename}", self.mariadb_domain_tb_name)
 
-        server_name_list = []
         server_ip_list = []
 
         try:
@@ -94,14 +98,16 @@ class IvacyvpnServerip(VpnDetector):
 
         if query_result:
             for row in query_result:
-                server_name_list.append(row[0])
+                self.server_name_list.append(row[0])
 
-        # add dc-xxx.pointtoserver.com to server_name_list
-        server_name_list.extend([f"dc-{str(index)}.pointtoserver.com" for index in range(1000)])
-        server_name_list = list(set(server_name_list))
+        # add dc-xxx.pointtoserver.com to self.server_name_list
+        self.server_name_list.extend([f"dc-{str(index)}.pointtoserver.com" for index in range(1000)])
+        self.server_name_list = list(set(self.server_name_list))
 
         if self.config['common']['active_scan']['switch'] and check_internet():
-            server_ip_list = self.resolve_dns_for_domain_list(server_name_list)
+            server_ip_list = self.resolve_dns_for_domain_list(self.server_name_list,
+                                                              max_workers=self.config['common']['active_scan']['max_workers'],
+                                                              max_calls_per_second=self.config['common']['active_scan']['max_calls_per_sec'])
             if server_ip_list:
                 server_ip_list = list(set(server_ip_list))
                 self.logger.info(
diff --git a/detection/vpnservices/turbovpn.py b/detection/vpnservices/turbovpn.py
index 2c5ab87..0266cd2 100644
--- a/detection/vpnservices/turbovpn.py
+++ b/detection/vpnservices/turbovpn.py
@@ -43,8 +43,8 @@ class Turbovpn(VpnDetector):
         self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name)
         self.sql = self.sql.replace("{$time_filter}", time_filter)
 
-        self.sql = self.sql.replace("{$security_table_name}", self.plugin_config['security_table_name'])\
-            .replace("{$security_policy_id}", str(self.plugin_config['security_policy_id']))
+        # self.sql = self.sql.replace("{$security_table_name}", self.plugin_config['security_table_name'])\
+        #     .replace("{$security_policy_id}", str(self.plugin_config['security_policy_id']))
 
         self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_name, self.sql))
 
@@ -55,7 +55,9 @@ class Turbovpn(VpnDetector):
             self.client.disconnect()
 
         if turbovpn_serverip_df.empty:
-            self.logger.error('[{}] - No turbovpn server ip found from security event. Policy id: {}.'.format(self.plugin_name, self.plugin_config['security_policy_id']))
+            # self.logger.error('[{}] - No turbovpn server ip found from security event. Policy id: {}.'.format(self.plugin_name, self.plugin_config['security_policy_id']))
+            self.logger.error(
+                '[{}] - No turbovpn server ip found from session record'.format(self.plugin_name))
             return []
         turbovpn_serverip_list = turbovpn_serverip_df[0].drop_duplicates().tolist()
         self.logger.info('[{}] - Query turbovpn server ip from clickhouse database successfully. {} items found'
diff --git a/detection/vpnservices/windscribevpn.py b/detection/vpnservices/windscribevpn.py
index 3676232..751228a 100644
--- a/detection/vpnservices/windscribevpn.py
+++ b/detection/vpnservices/windscribevpn.py
@@ -37,13 +37,19 @@ class Windscribevpn(VpnDetector):
         """
         result_group = []
 
+        result_group = []
+
         # start finding windscribevpn server name
-        windscribevpn_detector = WindscribevpnServername(self.start_time, self.end_time)
-        result_group.extend(windscribevpn_detector.find_server())
+        windscribevpn_server_name_detector = WindscribevpnServername(self.start_time, self.end_time)
+        server_name_object = windscribevpn_server_name_detector.find_server()
+        result_group.extend(server_name_object)
 
         # start finding windscribevpn server ip
-        windscribevpn_detector = WindscribevpnServerip()
-        result_group.extend(windscribevpn_detector.find_server())
+        windscribevpn_server_ip_detector = WindscribevpnServerip()
+        # server_name_list 初始化本周期已查询到的
+        windscribevpn_server_ip_detector.server_name_list = server_name_object[0].server_list if len(
+            server_name_object) > 0 else []
+        result_group.extend(windscribevpn_server_ip_detector.find_server())
 
         return result_group
 
@@ -73,6 +79,8 @@ class WindscribevpnServerip(VpnDetector):
         self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name']
         self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name']
 
+        self.server_name_list = []
+
 
     def find_more_servernames(self, server_name_list):
         """
@@ -119,11 +127,18 @@ class WindscribevpnServerip(VpnDetector):
         if query_result:
             servername_list = [i[0] for i in query_result]
 
+        self.server_name_list.extend(servername_list)
+
         # 判断是否能够访问外网，如果能够访问外网，则从外网获取windscribevpn_servername_list的域名解析地址
         if self.config['common']['active_scan']['switch'] and check_internet():
-            servername_list = self.find_more_servernames(servername_list)
+            servername_list = self.find_more_servernames(self.server_name_list)
             if len(servername_list) > 0:
-                resolved_ip_list = self.resolve_dns_for_domain_list(servername_list)
+                resolved_ip_list = self.resolve_dns_for_domain_list(self.server_name_list,
+                                                                  max_workers=self.config['common']['active_scan'][
+                                                                      'max_workers'],
+                                                                  max_calls_per_second=
+                                                                  self.config['common']['active_scan'][
+                                                                      'max_calls_per_sec'])
                 self.logger.info(
                     '[{}] - Get {} server ip by resolving server name successfully.'.format(
                         self.plugin_name, len(resolved_ip_list)))