diff options
Diffstat (limited to 'detection')
| -rw-r--r-- | detection/knowledgebase_monitor.py | 44 | ||||
| -rw-r--r-- | detection/tool/KnowledgeBaseTool.py | 10 | ||||
| -rw-r--r-- | detection/tool/__pycache__/KnowledgeBaseTool.cpython-39.pyc | bin | 0 -> 3537 bytes | |||
| -rw-r--r-- | detection/vpn_detector.py | 66 | ||||
| -rw-r--r-- | detection/vpnservices/cyberghostvpn.py | 8 | ||||
| -rw-r--r-- | detection/vpnservices/ipvanishvpn.py | 8 | ||||
| -rw-r--r-- | detection/vpnservices/ivacyvpn.py | 8 | ||||
| -rw-r--r-- | detection/vpnservices/windscribevpn.py | 8 |
8 files changed, 72 insertions, 80 deletions
diff --git a/detection/knowledgebase_monitor.py b/detection/knowledgebase_monitor.py index 10ecbdf..0a0dbb9 100644 --- a/detection/knowledgebase_monitor.py +++ b/detection/knowledgebase_monitor.py @@ -19,9 +19,9 @@ class KnowledgeBaseMonitor: self.knowledgebase_config = Config().config['knowledgebase'] self.knowledgebase_tool = KnowledgeApi(self.knowledgebase_config) - def get_vpn_count(self, vpn_service=None, start_t=None, end_t=None, node_type='ip', mode='active', timezone_gap_hour=0): + def get_vpn_count(self, plugin_name=None, start_t=None, end_t=None, node_type='ip', mode='active', timezone_gap_hour=0): """ - :param vpn_service: filter by vpn_service + :param plugin_name: filter by vpn_service :param node_type: 'ip' or 'domain' :param mode: 'active' or 'new' :param start_t: time range, format as '2024-01-20 15:00:00' @@ -30,11 +30,11 @@ class KnowledgeBaseMonitor: """ q = "" - # query node type + library_id = self.knowledgebase_tool.get_library_id(self.knowledgebase_config['library_name']) if node_type == 'ip': - library_id = self.knowledgebase_tool.get_library_id(self.knowledgebase_config['ip_library_name']) + q += "type = 'IP'" elif node_type == 'domain': - library_id = self.knowledgebase_tool.get_library_id(self.knowledgebase_config['domain_library_name']) + q += "type = 'Domain'" else: raise ValueError( "Wrong parameter \"node_type\" provided for KnowledgeBaseMonitor.get_vpn_ip_count: {}".format( @@ -42,16 +42,16 @@ class KnowledgeBaseMonitor: # query active node or new node if mode == 'active': - time_column = 'op_time' + time_column = 'updated_time' elif mode == 'new': - time_column = 'c_time' + time_column = 'created_time' else: raise ValueError( "Wrong parameter \"\mode\" provided for KnowledgeBaseMonitor. get_vpn_ip_count: {}".format(mode)) # query specific vpn service - if vpn_service and vpn_service!='all': - q += " and vpn_service_name = '{}'".format(vpn_service) + if plugin_name and plugin_name != 'all': + q += " and source_name = '{}'".format(plugin_name) # query specific time range if start_t: @@ -77,13 +77,13 @@ class KnowledgeBaseMonitor: # cycle active kb_metric['active_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap, - start_t=start_time, end_t=end_time, vpn_service=vpn_service) + start_t=start_time, end_t=end_time, plugin_name=vpn_service) kb_metric['new_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='new', timezone_gap_hour=time_zone_gap, - start_t=start_time, end_t=end_time, vpn_service=vpn_service) + start_t=start_time, end_t=end_time, plugin_name=vpn_service) kb_metric['active_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap, - start_t=start_time, end_t=end_time, vpn_service=vpn_service) + start_t=start_time, end_t=end_time, plugin_name=vpn_service) kb_metric['new_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='new', timezone_gap_hour=time_zone_gap, - start_t=start_time, end_t=end_time, vpn_service=vpn_service) + start_t=start_time, end_t=end_time, plugin_name=vpn_service) for key in monitor_result_dict.keys(): @@ -109,25 +109,25 @@ class KnowledgeBaseMonitor: if vpn_service is None: vpn_service = 'all' # all - kb_metric['ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', timezone_gap_hour=time_zone_gap, vpn_service=vpn_service) - kb_metric['domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', timezone_gap_hour=time_zone_gap, vpn_service=vpn_service) + kb_metric['ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', timezone_gap_hour=time_zone_gap, plugin_name=vpn_service) + kb_metric['domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', timezone_gap_hour=time_zone_gap, plugin_name=vpn_service) # all active in like 7 days t = (datetime.datetime.now().replace(minute=0, second=0, microsecond=0) - datetime.timedelta(days=self.monitor_config['outdated_days'])).strftime("%Y-%m-%d %H:%M:%S") - kb_metric['active_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap, start_t=t, vpn_service=vpn_service) - kb_metric['active_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap, start_t=t, vpn_service=vpn_service) + kb_metric['active_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap, start_t=t, plugin_name=vpn_service) + kb_metric['active_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap, start_t=t, plugin_name=vpn_service) # cycle active kb_metric['cycle_active_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='active', timezone_gap_hour=time_zone_gap, - start_t=current_start_time, end_t=current_end_time, vpn_service=vpn_service) + start_t=current_start_time, end_t=current_end_time, plugin_name=vpn_service) kb_metric['cycle_new_ip_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='ip', mode='new', timezone_gap_hour=time_zone_gap, - start_t=current_start_time, end_t=current_end_time, vpn_service=vpn_service) + start_t=current_start_time, end_t=current_end_time, plugin_name=vpn_service) kb_metric['cycle_active_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='active', timezone_gap_hour=time_zone_gap, - start_t=current_start_time, end_t=current_end_time, vpn_service=vpn_service) + start_t=current_start_time, end_t=current_end_time, plugin_name=vpn_service) kb_metric['cycle_new_domain_count{{type="{}"}}'.format(vpn_service)] = self.get_vpn_count(node_type='domain', mode='new', timezone_gap_hour=time_zone_gap, - start_t=current_start_time, end_t=current_end_time, vpn_service=vpn_service) + start_t=current_start_time, end_t=current_end_time, plugin_name=vpn_service) # churn ratio = (# new in current cycle)/(# all active) @@ -190,7 +190,7 @@ if __name__ == '__main__': for item in monitor_result_dict.items(): with open(monitor_file, "a") as file: file.write(item[0] + ' ' + str(item[1]) + '\n') - logger.info("[Monitor] {}~{} -{} {}".format(start_time, end_time, item[0], str(item[1]))) + logger.info("[Monitor] {}~{} - {} {}".format(start_time, end_time, item[0], str(item[1]))) # offline onetime mode diff --git a/detection/tool/KnowledgeBaseTool.py b/detection/tool/KnowledgeBaseTool.py index 0e2f3d1..a8079d4 100644 --- a/detection/tool/KnowledgeBaseTool.py +++ b/detection/tool/KnowledgeBaseTool.py @@ -14,6 +14,7 @@ import sys sys.path.append('..') from tool.LoggingTool import Logger +from urllib.parse import quote logger = Logger().getLogger() simplefilter(action='ignore', category=FutureWarning) @@ -27,7 +28,7 @@ class KnowledgeApi: self.api_path = config['api_path'] self.retry_max = config['api_retry_times'] self.request_timeout = config['api_timeout'] - # self.api_token = self.get_api_token() + self.api_token = config['api_token'] def get_api_token(self): @@ -59,13 +60,12 @@ class KnowledgeApi: sys.exit() - def file_import(self, file_path, knowledge_id, action, description=''): + def file_import(self, file_path, action, description=''): url = 'http://' + self.api_address + self.api_path file = open(file_path, "rb") file_object = {"file": file} param = { - "knowledgeId": knowledge_id, "action": action, "description": description } @@ -102,12 +102,14 @@ class KnowledgeApi: def get_knowledgebase_count(self, knowledge_id, page_size=None, page_no=None, q=None): url = 'http://' + self.api_address + '/v1/knowledgeBase/' + str(knowledge_id) + q = quote(q, 'utf-8') param = { - "q": q, "pageNo": page_no, "pageSize": page_size } + url += ('?q=' + q) + header = { "Cn-Authorization": self.api_token } diff --git a/detection/tool/__pycache__/KnowledgeBaseTool.cpython-39.pyc b/detection/tool/__pycache__/KnowledgeBaseTool.cpython-39.pyc Binary files differnew file mode 100644 index 0000000..6ae2542 --- /dev/null +++ b/detection/tool/__pycache__/KnowledgeBaseTool.cpython-39.pyc diff --git a/detection/vpn_detector.py b/detection/vpn_detector.py index 36ec6e4..d5de299 100644 --- a/detection/vpn_detector.py +++ b/detection/vpn_detector.py @@ -74,7 +74,7 @@ class VpnDetector: config = Config().config return config - def save_to_knowledgebase(self, object_list, object_type, vpn_service_name, plugin_id, plugin_name, output_filename, confidence='suspected'): + def save_results(self, object_list, object_type, vpn_service_name, plugin_id, plugin_name, output_filename, confidence='suspected'): """ Write data to local file and knowledge base :param object_type: ip or domain @@ -86,27 +86,25 @@ class VpnDetector: :param confidence: 3 kinds of confidence level, confirmed, suspect, tentative :return: """ - if object_type == 'ip': - library_name = self.config['knowledgebase']['ip_library_name'] - else: - library_name = self.config['knowledgebase']['domain_library_name'] - knowledge_id = self.kb.get_library_id(library_name) # convert result data into required format https://docs.geedge.net/pages/viewpage.action?pageId=104760257 - result_df = pd.DataFrame() + result_df = pd.DataFrame(columns=['type', 'source_id', 'source_name', 'ip_addr_format', 'ip1', 'ip2', 'domain', 'method', 'confidence', 'tags', 'description', 'first_seen_time', 'last_seen_time']) + if object_type == 'ip': result_df['ip1'] = object_list result_df['ip2'] = object_list - result_df.insert(0, 'addr_format', 'Single') + result_df['type'] = 'IP' + result_df['ip_addr_format'] = 'Single' + if object_type == 'domain': - result_df['domain'] = object_list + result_df['domain'] = ['$'+i for i in object_list] + result_df['type'] = 'Domain' - result_df['plugin_id'] = plugin_id - result_df['plugin_name'] = plugin_name - result_df['vpn_service_name'] = vpn_service_name + result_df['source_id'] = plugin_id + result_df['source_name'] = plugin_name + result_df['tags'] = vpn_service_name result_df['method'] = 'passive_ml' result_df['confidence'] = confidence - result_df['is_valid'] = 1 # result save if len(result_df) > 0: @@ -124,7 +122,9 @@ class VpnDetector: knowledge_api = KnowledgeApi(self.config['knowledgebase']) self.logger.info('[{}] - [Updating knowledgebase]- {} num:{}'.format(plugin_name, object_type, len(object_list))) description_str = "Update {} record(s).".format(len(object_list)) - knowledge_api.file_import(result_file, knowledge_id, 'update', description_str) + knowledge_api.file_import(result_file, 'update', description_str) + self.logger.info( + '[{}] - Write to knowledgebase successfully.'.format(plugin_name)) @@ -201,9 +201,7 @@ class VpnDetector: return [i for i in original_ip_list if ':' not in i] -# 入口函数定义 -if __name__ == '__main__': - +def main(): parser = argparse.ArgumentParser(description='VPN detection') parser.add_argument('-p', '--plugin', type=str, help='plugin name') parser.add_argument('-m', '--mode', type=str, default='recent', help='recent or fixed') @@ -232,7 +230,8 @@ if __name__ == '__main__': exit() # 根据当前时间向前取整小时 end_time = datetime.datetime.now().strftime("%Y-%m-%d %H:00:00") - start_time = (datetime.datetime.now() - datetime.timedelta(hours=recent_interval)).strftime("%Y-%m-%d %H:00:00") + start_time = (datetime.datetime.now() - datetime.timedelta(hours=recent_interval)).strftime( + "%Y-%m-%d %H:00:00") elif mode == 'fixed': if start_time == '' or end_time == '': print('Please input correct time format') @@ -242,7 +241,6 @@ if __name__ == '__main__': print('Please input correct time mode') exit() - detector = None if plugin_name == 'hotspotvpn': from vpnservices.hotspotvpn import Hotspotvpn @@ -275,31 +273,31 @@ if __name__ == '__main__': print('Please input correct plugin name') exit() - result_group = detector.find_server() for server_group in result_group: result_list = server_group.server_list detector.logger.debug('[{}] - result list before filter: {}.'.format(detector.plugin_name, result_list)) if server_group.object_type == 'ip': # 日志记录IP数量 - detector.logger.info('[{}] - {} {} found.'.format(detector.plugin_name, len(result_list), server_group.object_type)) + detector.logger.info( + '[{}] - {} {} found.'.format(detector.plugin_name, len(result_list), server_group.object_type)) result_list = detector.filtered_by_isp(result_list, detector.config['common']['protected_isp_list']) result_list = detector.filtered_by_ip(result_list, detector.config['common']['protected_ip_list']) result_list = detector.filter_ipv6(result_list) detector.logger.debug( - '[{}] - filtered list {}.'.format(detector.plugin_name, [i for i in server_group.server_list if i not in result_list])) - detector.logger.info('[{}] - Filtered by ISP and IP, {} {} left.'.format(detector.plugin_name, len(result_list), server_group.object_type)) - if len(result_list) > 0: - detector.save_to_knowledgebase(result_list, server_group.object_type, - detector.vpn_service_name, detector.plugin_id, - detector.plugin_name, - server_group.output_file_name, - detector.confidence) - - # Output logs to record the amount of data written, plugin id, plugin name and other information + '[{}] - filtered list {}.'.format(detector.plugin_name, + [i for i in server_group.server_list if i not in result_list])) detector.logger.info( - '[{}] - Write {} {} to knowledgebase successfully.'.format(detector.plugin_name, - len(result_list), - server_group.object_type)) + '[{}] - Filtered by ISP and IP, {} {} left.'.format(detector.plugin_name, len(result_list), + server_group.object_type)) + if len(result_list) > 0: + detector.save_results(result_list, server_group.object_type, + detector.vpn_service_name, detector.plugin_id, + detector.plugin_name, + server_group.output_file_name, + detector.confidence) +# 入口函数定义 +if __name__ == '__main__': + main()
\ No newline at end of file diff --git a/detection/vpnservices/cyberghostvpn.py b/detection/vpnservices/cyberghostvpn.py index b68adb1..5c90de8 100644 --- a/detection/vpnservices/cyberghostvpn.py +++ b/detection/vpnservices/cyberghostvpn.py @@ -71,14 +71,12 @@ class CyberghostvpnServerip(VpnDetector): self.kb_sql = self.plugin_config['ip']['kb_sql'] self.kb_dbname = self.config['knowledgebase']['db_name'] - self.kb_table_name = self.config['knowledgebase']['domain_library_name'] self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), self.config['mariadb']['db_name']) self.mariadb_dbname = self.config['mariadb']['db_name'] - self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] - self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] + self.mariadb_tb_name = self.config['mariadb']['table_name'] self.sql = self.plugin_config['ip']['sql'] @@ -130,7 +128,7 @@ class CyberghostvpnServerip(VpnDetector): :return: cyberghostvpn server ip list """ self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace( - "{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) + "{$mariadb_tablename}", self.mariadb_tb_name) # 根据server name获取ip self.logger.info('[{}] - Get servername from knowledge base.'.format(self.plugin_name)) @@ -142,7 +140,7 @@ class CyberghostvpnServerip(VpnDetector): self.mariadb.close() if query_result: - servername_list = [i[0] for i in query_result] + servername_list = [i[0].strip('$') for i in query_result] self.server_name_list.extend(servername_list) # 判断是否能够访问外网，如果能够访问外网，则从外网获取cyberghost_servername_list的域名解析地址 diff --git a/detection/vpnservices/ipvanishvpn.py b/detection/vpnservices/ipvanishvpn.py index 1b1d5fb..6f2db93 100644 --- a/detection/vpnservices/ipvanishvpn.py +++ b/detection/vpnservices/ipvanishvpn.py @@ -71,14 +71,12 @@ class IpvanishvpnServerip(VpnDetector): self.kb_sql = self.plugin_config['ip']['kb_sql'] self.kb_dbname = self.config['knowledgebase']['db_name'] - self.kb_table_name = self.config['knowledgebase']['domain_library_name'] self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), self.config['mariadb']['db_name']) self.mariadb_dbname = self.config['mariadb']['db_name'] - self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] - self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] + self.mariadb_tb_name = self.config['mariadb']['table_name'] self.server_name_list = [] @@ -110,7 +108,7 @@ class IpvanishvpnServerip(VpnDetector): Get ipvanishvpn server ip by resolving ipvanishvpn server name :return: ipvanishvpn server ip list """ - self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) + self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_tablename}", self.mariadb_tb_name) servername_list = [] resolved_ip_list = [] @@ -120,7 +118,7 @@ class IpvanishvpnServerip(VpnDetector): self.mariadb.close() if query_result: - servername_list = [i[0] for i in query_result] + servername_list = [i[0].strip('$') for i in query_result] self.server_name_list.extend(servername_list) diff --git a/detection/vpnservices/ivacyvpn.py b/detection/vpnservices/ivacyvpn.py index e12fd9c..9929583 100644 --- a/detection/vpnservices/ivacyvpn.py +++ b/detection/vpnservices/ivacyvpn.py @@ -68,14 +68,12 @@ class IvacyvpnServerip(VpnDetector): self.kb_sql = self.plugin_config['ip']['kb_sql'] self.kb_dbname = self.config['knowledgebase']['db_name'] - self.kb_table_name = self.config['knowledgebase']['domain_library_name'] self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), self.config['mariadb']['db_name']) self.mariadb_dbname = self.config['mariadb']['db_name'] - self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] - self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] + self.mariadb_tb_name = self.config['mariadb']['table_name'] self.server_name_list = [] @@ -87,7 +85,7 @@ class IvacyvpnServerip(VpnDetector): Find ivacyvpn server ip by resolving ivacyvpn server name :return: server ip list """ - self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) + self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_tablename}", self.mariadb_tb_name) server_ip_list = [] @@ -98,7 +96,7 @@ class IvacyvpnServerip(VpnDetector): if query_result: for row in query_result: - self.server_name_list.append(row[0]) + self.server_name_list.append(row[0].strip('$')) # add dc-xxx.pointtoserver.com to self.server_name_list self.server_name_list.extend([f"dc-{str(index)}.pointtoserver.com" for index in range(1000)]) diff --git a/detection/vpnservices/windscribevpn.py b/detection/vpnservices/windscribevpn.py index 5978907..43cee14 100644 --- a/detection/vpnservices/windscribevpn.py +++ b/detection/vpnservices/windscribevpn.py @@ -79,14 +79,12 @@ class WindscribevpnActiveServerip(VpnDetector): self.kb_sql = self.plugin_config['ip']['kb_sql'] self.sql = self.plugin_config['ip']['sql'] self.kb_dbname = self.config['knowledgebase']['db_name'] - self.kb_table_name = self.config['knowledgebase']['domain_library_name'] self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), self.config['mariadb']['db_name']) self.mariadb_dbname = self.config['mariadb']['db_name'] - self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] - self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] + self.mariadb_tb_name = self.config['mariadb']['table_name'] self.server_name_list = [] @@ -122,7 +120,7 @@ class WindscribevpnActiveServerip(VpnDetector): :return: windscribevpn server ip list """ self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace( - "{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) + "{$mariadb_tablename}", self.mariadb_tb_name) servername_list = [] resolved_ip_list = [] @@ -132,7 +130,7 @@ class WindscribevpnActiveServerip(VpnDetector): self.mariadb.close() if query_result: - servername_list = [i[0] for i in query_result] + servername_list = [i[0].strip('$') for i in query_result] self.server_name_list.extend(servername_list) |