diff options
Diffstat (limited to 'detection/vpnservices')
17 files changed, 799 insertions, 688 deletions
diff --git a/detection/vpnservices/cyberghostvpn.py b/detection/vpnservices/cyberghostvpn.py new file mode 100644 index 0000000..09ba1af --- /dev/null +++ b/detection/vpnservices/cyberghostvpn.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2024/1/31 15:03 +# @author : yinjinagyi +# @File : cyberghostvpn.py +# @Function: +import datetime +import re + +import pandas as pd + +from tool.Functions import check_internet +from tool.MariadbTool import MariadbUtil +from vpn_detector import VpnDetector, ServerGroup + + +class Cyberghostvpn(VpnDetector): + """ + + This class is used to detect cyberghostvpn server ip and server name + """ + + def __init__(self, start_time, end_time): + super().__init__(start_time, end_time) + self.plugin_config = self.load_config()['cyberghostvpn'] + self.vpn_service_name = self.plugin_config['vpn_service_name'] + self.plugin_name = self.plugin_config['plugin_name'] + self.plugin_id = self.plugin_config['plugin_id'] + self.confidence = self.plugin_config['confidence'] + self.start_time = start_time + self.end_time = end_time + + + def find_server(self): + """ + Get cyberghostvpn server ip and server name from clickhouse database + :return: cyberghostvpn server ip list and server name list + """ + result_group = [] + + # start finding cyberghostvpn server name + cyberghostvpn_detector = CyberghostvpnServername(self.start_time, self.end_time) + result_group.extend(cyberghostvpn_detector.find_server()) + + # start finding cyberghostvpn server ip + cyberghostvpn_detector = CyberghostvpnServerip() + result_group.extend(cyberghostvpn_detector.find_server()) + + return result_group + + + +class CyberghostvpnServerip(VpnDetector): + """ + This class is used to detect cyberghostvpn server ip + """ + + def __init__(self): + super().__init__('', '') + self.plugin_config = self.load_config()['cyberghostvpn'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['ip']['object_type'] + # 开始时间为当前的整点时间 + self.start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:00:00") + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', + '_').replace( + ':', '')[:13] + '.csv' + + self.kb_sql = self.plugin_config['ip']['kb_sql'] + self.kb_dbname = self.config['knowledgebase']['db_name'] + self.kb_table_name = self.config['knowledgebase']['domain_library_name'] + + self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], + self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), + self.config['mariadb']['db_name']) + self.mariadb_dbname = self.config['mariadb']['db_name'] + self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] + self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] + + + def find_more_servernames(self, server_name_list): + """ + Find more server name from observed cyberghost server name list + :return: server name list + """ + pattern_list = [] + expanded_server_names = [] + + + for server_name in server_name_list: + pattern = re.compile(r'\.(.*?)\-rack') + pattern_list.append(pattern.findall(server_name)[0]) + pattern_list = set(pattern_list) + + for pattern_str in pattern_list: + domain_list = [f"blade{str(index1)}.{pattern_str}-rack4{str(index2).zfill(2)}.nodes.gen4.ninja" for index1 in range(1, 100) for index2 in range(1, 100)] + expanded_server_names.extend(domain_list) + + return expanded_server_names + + + def find_server(self): + """ + Get cyberghostvpn server ip by resolving cyberghostvpn server name + :return: cyberghostvpn server ip list + """ + self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace( + "{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) + + servername_list = [] + resolved_ip_list = [] + try: + query_result = self.mariadb.query_sql(self.kb_sql) + finally: + self.mariadb.close() + + if query_result: + servername_list = [i[0] for i in query_result] + + # 判断是否能够访问外网，如果能够访问外网，则从外网获取cyberghost_servername_list的域名解析地址 + if check_internet(): + servername_list = self.find_more_servernames(servername_list) + if len(servername_list) > 0: + resolved_ip_list = self.resolve_dns_for_domain_list(servername_list) + self.logger.info('[{}] - Get {} server ip by resolving server name successfully.'.format(self.plugin_name, len(resolved_ip_list))) + else: + self.logger.info( + '[{}] - No cyberghost server name found from knowledge database.'.format(self.plugin_name)) + else: + self.logger.info('[{}] - No internet connection, skip dns resolve.'.format(self.plugin_name)) + + return [ServerGroup(self.object_type, resolved_ip_list, self.output_file_name)] + + + +class CyberghostvpnServername(VpnDetector): + """ + + This class is used to detect cyberghostvpn server name + """ + + def __init__(self, start_time, end_time): + super().__init__(start_time, end_time) + self.plugin_config = self.load_config()['cyberghostvpn'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['domain']['object_type'] + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + self.start_time = start_time + self.end_time = end_time + + self.sql = self.plugin_config['domain']['sql'] + + def find_server(self): + """ + Get cyberghostvpn server name from clickhouse database + :return: cyberghostvpn server name list + """ + self.logger.info('[{}] - Start to query cyberghostvpn server name from session record'.format(self.plugin_name)) + + # construct query sql + TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', + self.config['common'][ + 'recv_time_columnname']) + time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( + self.end_time)).replace("{$time_zone}", self.time_zone) + self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) + self.sql = self.sql.replace("{$time_filter}", time_filter) + self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_config['plugin_name'], self.sql)) + + # query data from clickhouse database + try: + cyberghostvpn_servername_df = pd.DataFrame(self.client.execute(self.sql)) + finally: + self.client.disconnect() + + if cyberghostvpn_servername_df.empty: + self.logger.info('[{}] - No cyberghostvpn server name found from session records'.format(self.plugin_name)) + return [] + cyberghostvpn_servername_list = cyberghostvpn_servername_df[0].drop_duplicates().tolist() + self.logger.info('[{}] - Query cyberghostvpn server name from session records successfully. {} items found' + .format(self.plugin_name, len(cyberghostvpn_servername_list))) + + return [ServerGroup(self.object_type, cyberghostvpn_servername_list, self.output_file_name)] diff --git a/detection/vpnservices/cyberghostvpn_serverip.py b/detection/vpnservices/cyberghostvpn_serverip.py deleted file mode 100644 index 4505a95..0000000 --- a/detection/vpnservices/cyberghostvpn_serverip.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2024/1/16 20:01 -# @author : yinjinagyi -# @File : cyberghostvpn_serverip.py.py -# @Function: -import re - -from vpn_detector import VpnDetector -from tool.Functions import check_internet -from tool.MariadbTool import MariadbUtil - - -class CyberghostvpnServerip(VpnDetector): - """ - This class is used to detect cyberghostvpn server ip - """ - - def __init__(self): - super().__init__('', '') - self.plugin_config = self.load_config()['cyberghostvpn_serverip'] - self.plugin_id = self.plugin_config['plugin_id'] - self.plugin_name = self.plugin_config['plugin_name'] - self.object_type = self.plugin_config['object_type'] - self.vpn_service_name = self.plugin_config['vpn_service_name'] - self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' - - self.kb_sql = self.plugin_config['kb_sql'] - self.kb_dbname = self.config['knowledgebase']['db_name'] - self.kb_table_name = self.config['knowledgebase']['domain_library_name'] - - self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], - self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), - self.config['mariadb']['db_name']) - self.mariadb_dbname = self.config['mariadb']['db_name'] - self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] - self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] - - - def find_more_servernames(self, server_name_list): - """ - Find more server name from observed cyberghost server name list - :return: server name list - """ - pattern_list = [] - expanded_server_names = [] - - - for server_name in server_name_list: - pattern = re.compile(r'\.(.*?)\-rack') - pattern_list.append(pattern.findall(server_name)[0]) - pattern_list = set(pattern_list) - - for pattern_str in pattern_list: - domain_list = [f"blade{str(index1)}.{pattern_str}-rack4{str(index2).zfill(2)}.nodes.gen4.ninja" for index1 in range(1, 100) for index2 in range(1, 100)] - expanded_server_names.extend(domain_list) - - return expanded_server_names - - - def find_server(self): - """ - Get cyberghostvpn server ip by resolving cyberghostvpn server name - :return: cyberghostvpn server ip list - """ - self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace( - "{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) - - servername_list = [] - resolved_ip_list = [] - try: - query_result = self.mariadb.query_sql(self.kb_sql) - finally: - self.mariadb.close() - - if query_result: - servername_list = [i[0] for i in query_result] - - # 判断是否能够访问外网，如果能够访问外网，则从外网获取cyberghost_servername_list的域名解析地址 - if check_internet(): - servername_list = self.find_more_servernames(servername_list) - if len(servername_list) > 0: - resolved_ip_list = self.resolve_dns_for_domain_list(servername_list) - else: - self.logger.info('No cyberghost server name found from knowledge database.') - else: - self.logger.info('Failed to resolve cyberghost vpn servername. Cannot access internet.') - - return resolved_ip_list - - - - - - diff --git a/detection/vpnservices/cyberghostvpn_servername.py b/detection/vpnservices/cyberghostvpn_servername.py deleted file mode 100644 index 2d9c277..0000000 --- a/detection/vpnservices/cyberghostvpn_servername.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2024/1/16 19:51 -# @author : yinjinagyi -# @File : cyberghostvpn_servername.py.py -# @Function: - -import sys -sys.path.append('..') -from vpn_detector import VpnDetector -import pandas as pd - - -class CyberghostvpnServername(VpnDetector): - """ - - This class is used to detect cyberghostvpn server name - """ - - def __init__(self, start_time, end_time): - super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['cyberghostvpn_servername'] - self.plugin_id = self.plugin_config['plugin_id'] - self.plugin_name = self.plugin_config['plugin_name'] - self.object_type = self.plugin_config['object_type'] - self.vpn_service_name = self.plugin_config['vpn_service_name'] - self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' - self.start_time = start_time - self.end_time = end_time - - self.sql = self.plugin_config['sql'] - - def find_server(self): - """ - Get cyberghostvpn server name from clickhouse database - :return: cyberghostvpn server name list - """ - self.logger.info('Start to query cyberghostvpn server name from session record') - - # construct query sql - TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', self.config['common']['recv_time_columnname']) - time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( - self.end_time)).replace("{$time_zone}", self.time_zone) - self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) - self.sql = self.sql.replace("{$time_filter}", time_filter) - self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) - - # query data from clickhouse database - try: - cyberghostvpn_servername_df = pd.DataFrame(self.client.execute(self.sql)) - finally: - self.client.disconnect() - - if cyberghostvpn_servername_df.empty: - self.logger.info('No cyberghostvpn server name found from session records') - return [] - cyberghostvpn_servername_list = cyberghostvpn_servername_df[0].drop_duplicates().tolist() - self.logger.info('Query cyberghostvpn server name from session records successfully. {} items found' - .format(len(cyberghostvpn_servername_list))) - return cyberghostvpn_servername_list diff --git a/detection/vpnservices/geckovpn_serverip.py b/detection/vpnservices/geckovpn.py index 1c607d3..69e66ce 100644 --- a/detection/vpnservices/geckovpn_serverip.py +++ b/detection/vpnservices/geckovpn.py @@ -2,26 +2,28 @@ # -*- coding: utf-8 -*- # @Time : 2024/1/17 14:20 # @author : yinjinagyi -# @File : geckovpn_serverip.py +# @File : geckovpn.py # @Function: import pandas as pd -from vpn_detector import VpnDetector +from vpn_detector import VpnDetector, ServerGroup -class GeckovpnServerip(VpnDetector): + +class Geckovpn(VpnDetector): """ This class is used to detect geckovpn server ip """ def __init__(self, start_time, end_time): super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['geckovpn_serverip'] + self.plugin_config = self.load_config()['geckovpn'] self.plugin_id = self.plugin_config['plugin_id'] self.plugin_name = self.plugin_config['plugin_name'] self.object_type = self.plugin_config['object_type'] self.vpn_service_name = self.plugin_config['vpn_service_name'] self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ','_').replace(':', '')[:13] + '.csv' + self.start_time = start_time self.end_time = end_time @@ -32,7 +34,7 @@ class GeckovpnServerip(VpnDetector): Get geckovpn server ip from clickhouse database :return: geckovpn server ip list """ - self.logger.info('Start to query geckovpn server ip from session records') + self.logger.info('[{}] - Start to query server ip from session records'.format(self.plugin_name)) # construct query sql TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', self.config['common']['recv_time_columnname']) @@ -40,7 +42,7 @@ class GeckovpnServerip(VpnDetector): self.end_time)).replace("{$time_zone}", self.time_zone) self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) self.sql = self.sql.replace("{$time_filter}", time_filter) - self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) + self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_name, self.sql)) # query data from clickhouse database try: @@ -49,9 +51,11 @@ class GeckovpnServerip(VpnDetector): self.client.disconnect() if geckovpn_serverip_df.empty: - self.logger.info('No geckovpn server ip found from session records') + self.logger.info('[{}] - No server ip found from session records'.format(self.plugin_name)) return [] geckovpn_serverip_list = geckovpn_serverip_df[0].drop_duplicates().tolist() - self.logger.info('Query geckovpn server ip from session records successfully. {} items found' - .format(len(geckovpn_serverip_list))) - return geckovpn_serverip_list + self.logger.info('[{}] - Query server ip from clickhouse database successfully. {} items found' + .format(self.plugin_name, len(geckovpn_serverip_list))) + + + return [ServerGroup(object_type='ip', server_list=geckovpn_serverip_list, output_file_name=self.output_file_name)] diff --git a/detection/vpnservices/hotspotvpn_serverip.py b/detection/vpnservices/hotspotvpn.py index 2c853bf..d28976d 100644 --- a/detection/vpnservices/hotspotvpn_serverip.py +++ b/detection/vpnservices/hotspotvpn.py @@ -2,14 +2,14 @@ # -*- coding: utf-8 -*- # @Time : 2024/1/11 15:45 # @author : yinjinagyi -# @File : hotspotvpn_serverip.py +# @File : hotspotvpn.py # @Function: -from vpn_detector import VpnDetector +from vpn_detector import VpnDetector, ServerGroup import pandas as pd -class HotspotvpnServerip(VpnDetector): +class Hotspotvpn(VpnDetector): """ This class is used to detect hotspotvpn server ip @@ -17,13 +17,13 @@ class HotspotvpnServerip(VpnDetector): def __init__(self, start_time, end_time): super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['hotspotvpn_serverip'] + self.plugin_config = self.load_config()['hotspotvpn'] self.plugin_id = self.plugin_config['plugin_id'] self.plugin_name = self.plugin_config['plugin_name'] self.object_type = self.plugin_config['object_type'] self.vpn_service_name = self.plugin_config['vpn_service_name'] self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' self.start_time = start_time self.end_time = end_time @@ -33,9 +33,9 @@ class HotspotvpnServerip(VpnDetector): def find_server(self): """ Get hotspotvpn server ip from clickhouse database - :return: hotspotvpn server ip list + :return: hotspotvpn server group """ - self.logger.info('Start to query hotspotvpn server ip from session records') + self.logger.info('[{}] - Start to query server ip from session records'.format(self.plugin_name)) # construct query sql TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', self.config['common']['recv_time_columnname']) @@ -45,7 +45,7 @@ class HotspotvpnServerip(VpnDetector): self.sql = self.sql.replace("{$time_filter}", time_filter) self.sql = self.sql.replace("{$domain_list}", ','.join(self.masquerede_domains)) - self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) + self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_name, self.sql)) # query data from clickhouse database try: @@ -54,10 +54,12 @@ class HotspotvpnServerip(VpnDetector): self.client.disconnect() if hotspotvpn_serverip_df.empty: - self.logger.info('No hotspotvpn server ip found from session records') + self.logger.info('[{}] - No server ip found from session records'.format(self.plugin_name)) return [] hotspotvpn_serverip_list = hotspotvpn_serverip_df[0].drop_duplicates().tolist() - self.logger.info('Query hotspotvpn server ip from clickhouse database successfully. {} items found' - .format(len(hotspotvpn_serverip_list))) + self.logger.info('[{}] - Query server ip from clickhouse database successfully. {} items found' + .format(self.plugin_name, len(hotspotvpn_serverip_list))) + + + return [ServerGroup(object_type='ip', server_list=hotspotvpn_serverip_list, output_file_name=self.output_file_name)] - return hotspotvpn_serverip_list diff --git a/detection/vpnservices/ipvanishvpn.py b/detection/vpnservices/ipvanishvpn.py new file mode 100644 index 0000000..88f8f1f --- /dev/null +++ b/detection/vpnservices/ipvanishvpn.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2024/1/30 18:20 +# @author : yinjinagyi +# @File : ipvanishvpn.py +# @Function: + +import sys +sys.path.append('..') +import datetime +import pandas as pd +from tool.Functions import check_internet +from tool.MariadbTool import MariadbUtil + +from vpn_detector import VpnDetector, ServerGroup + + +class Ipvanishvpn(VpnDetector): + """ + + This class is used to detect ipvanishvpn server ip and server name + """ + + def __init__(self, start_time, end_time): + super().__init__(start_time, end_time) + self.plugin_config = self.load_config()['ipvanishvpn'] + self.vpn_service_name = self.plugin_config['vpn_service_name'] + self.plugin_name = self.plugin_config['plugin_name'] + self.plugin_id = self.plugin_config['plugin_id'] + self.confidence = self.plugin_config['confidence'] + self.start_time = start_time + self.end_time = end_time + + + def find_server(self): + """ + Get ipvanishvpn server ip and server name from clickhouse database + :return: ipvanishvpn server ip list and server name list + """ + result_group = [] + + # start finding ipvanishvpn server name + ipvanishvpn_detector = IpvanishvpnServername(self.start_time, self.end_time) + result_group.extend(ipvanishvpn_detector.find_server()) + + # start finding ipvanishvpn server ip + ipvanishvpn_detector = IpvanishvpnServerip() + result_group.extend(ipvanishvpn_detector.find_server()) + + return result_group + + +class IpvanishvpnServerip(VpnDetector): + """ + This class is used to detect ipvanishvpn server ip + """ + + def __init__(self): + super().__init__('', '') + self.plugin_config = self.load_config()['ipvanishvpn'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['ip']['object_type'] + # 开始时间为当前的整点时间 + self.start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:00:00") + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + + + self.kb_sql = self.plugin_config['ip']['kb_sql'] + self.kb_dbname = self.config['knowledgebase']['db_name'] + self.kb_table_name = self.config['knowledgebase']['domain_library_name'] + + self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], + self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), + self.config['mariadb']['db_name']) + self.mariadb_dbname = self.config['mariadb']['db_name'] + self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] + self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] + + + def find_more_servernames(self, server_name_list): + """ + Find more server name from observed ipvanish server name list + :return: server name list + """ + prefix_list = [] + expanded_server_names = [] + + for server_name in server_name_list: + domain = server_name.strip() + domain_prefix = domain[:5] + prefix_list.append(domain_prefix) + + prefix_list = set(prefix_list) + + for domain_prefix in prefix_list: + domain_list = [f"{domain_prefix}{str(index).zfill(2)}.vpn.ipvanish.com" for index in range(100)] + expanded_server_names.extend(domain_list) + + return expanded_server_names + + + def find_server(self): + """ + Get ipvanishvpn server ip by resolving ipvanishvpn server name + :return: ipvanishvpn server ip list + """ + self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) + + servername_list = [] + resolved_ip_list = [] + try: + query_result = self.mariadb.query_sql(self.kb_sql) + finally: + self.mariadb.close() + + if query_result: + servername_list = [i[0] for i in query_result] + + # 判断是否能够访问外网，如果能够访问外网，则从外网获取ipvanish_servername_list的域名解析地址 + if check_internet(): + servername_list = self.find_more_servernames(servername_list) + if len(servername_list) > 0: + resolved_ip_list = self.resolve_dns_for_domain_list(servername_list) + self.logger.info( + '[{}] - Get {} server ip by resolving server name successfully.'.format( + self.plugin_name, len(resolved_ip_list))) + else: + self.logger.info('[{}] - No ipvanish server name found from knowledge database.'.format(self.plugin_name)) + else: + self.logger.info('[{}] - No internet connection, skip dns resolve.'.format(self.plugin_name)) + + return [ServerGroup(self.object_type, resolved_ip_list, self.output_file_name)] + + +class IpvanishvpnServername(VpnDetector): + """ + + This class is used to detect ipvanish server name + """ + + def __init__(self, start_time, end_time): + super().__init__(start_time, end_time) + self.plugin_config = self.load_config()['ipvanishvpn'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['domain']['object_type'] + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + self.start_time = start_time + self.end_time = end_time + + self.sql = self.plugin_config['domain']['sql'] + + def find_server(self): + """ + Get ipvanishvpn server name from clickhouse database + :return: ipvanishvpn server name list + """ + self.logger.info('[{}] - Start to query ipvanishvpn server name from session record'.format(self.plugin_name)) + + # construct query sql + TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', self.config['common']['recv_time_columnname']) + time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( + self.end_time)).replace("{$time_zone}", self.time_zone) + self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) + self.sql = self.sql.replace("{$time_filter}", time_filter) + self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_config['plugin_name'], self.sql)) + + # query data from clickhouse database + try: + ipvanishvpn_servername_df = pd.DataFrame(self.client.execute(self.sql)) + finally: + self.client.disconnect() + + if ipvanishvpn_servername_df.empty: + self.logger.info('[{}] - No ipvanishvpn server name found from session records'.format(self.plugin_name)) + return [] + ipvanishvpn_servername_list = ipvanishvpn_servername_df[0].drop_duplicates().tolist() + self.logger.info('[{}] - Query ipvanishvpn server name from session records successfully. {} items found' + .format(self.plugin_name, len(ipvanishvpn_servername_list))) + + return [ServerGroup(self.object_type, ipvanishvpn_servername_list, self.output_file_name)] diff --git a/detection/vpnservices/ipvanishvpn_serverip.py b/detection/vpnservices/ipvanishvpn_serverip.py deleted file mode 100644 index 9a26407..0000000 --- a/detection/vpnservices/ipvanishvpn_serverip.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2024/1/12 10:00 -# @author : yinjinagyi -# @File : ipvanishvpn_serverip.py -# @Function: - -from vpn_detector import VpnDetector -from tool.Functions import check_internet -from tool.MariadbTool import MariadbUtil - - -class IpvanishvpnServerip(VpnDetector): - """ - This class is used to detect ipvanishvpn server ip - """ - - def __init__(self): - super().__init__('', '') - self.plugin_config = self.load_config()['ipvanishvpn_serverip'] - self.plugin_id = self.plugin_config['plugin_id'] - self.plugin_name = self.plugin_config['plugin_name'] - self.object_type = self.plugin_config['object_type'] - self.vpn_service_name = self.plugin_config['vpn_service_name'] - self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' - - - self.kb_sql = self.plugin_config['kb_sql'] - self.kb_dbname = self.config['knowledgebase']['db_name'] - self.kb_table_name = self.config['knowledgebase']['domain_library_name'] - - self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], - self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), - self.config['mariadb']['db_name']) - self.mariadb_dbname = self.config['mariadb']['db_name'] - self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] - self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] - - - def find_more_servernames(self, server_name_list): - """ - Find more server name from observed ipvanish server name list - :return: server name list - """ - prefix_list = [] - expanded_server_names = [] - - for server_name in server_name_list: - domain = server_name.strip() - domain_prefix = domain[:5] - prefix_list.append(domain_prefix) - - prefix_list = set(prefix_list) - - for domain_prefix in prefix_list: - domain_list = [f"{domain_prefix}{str(index).zfill(2)}.vpn.ipvanish.com" for index in range(100)] - expanded_server_names.extend(domain_list) - - return expanded_server_names - - - def find_server(self): - """ - Get ipvanishvpn server ip by resolving ipvanishvpn server name - :return: ipvanishvpn server ip list - """ - self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) - - servername_list = [] - resolved_ip_list = [] - try: - query_result = self.mariadb.query_sql(self.kb_sql) - finally: - self.mariadb.close() - - if query_result: - servername_list = [i[0] for i in query_result] - - # 判断是否能够访问外网，如果能够访问外网，则从外网获取ipvanish_servername_list的域名解析地址 - if check_internet(): - servername_list = self.find_more_servernames(servername_list) - if len(servername_list) > 0: - resolved_ip_list = self.resolve_dns_for_domain_list(servername_list) - else: - self.logger.info('No ipvanish server name found from knowledge database.') - else: - self.logger.info('No internet connection, skip dns resolve.') - - return resolved_ip_list - - - diff --git a/detection/vpnservices/ipvanishvpn_servername.py b/detection/vpnservices/ipvanishvpn_servername.py deleted file mode 100644 index 85fd505..0000000 --- a/detection/vpnservices/ipvanishvpn_servername.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2024/1/11 15:45 -# @author : yinjinagyi -# @File : ipvanishvpn_servername.py -# @Function: - -import sys -sys.path.append('..') -from vpn_detector import VpnDetector -import pandas as pd - - -class IpvanishvpnServername(VpnDetector): - """ - - This class is used to detect ipvanish server name - """ - - def __init__(self, start_time, end_time): - super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['ipvanishvpn_servername'] - self.plugin_id = self.plugin_config['plugin_id'] - self.plugin_name = self.plugin_config['plugin_name'] - self.object_type = self.plugin_config['object_type'] - self.vpn_service_name = self.plugin_config['vpn_service_name'] - self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' - self.start_time = start_time - self.end_time = end_time - - self.sql = self.plugin_config['sql'] - - def find_server(self): - """ - Get ipvanishvpn server name from clickhouse database - :return: ipvanishvpn server name list - """ - self.logger.info('Start to query ipvanishvpn server name from session record') - - # construct query sql - TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', self.config['common']['recv_time_columnname']) - time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( - self.end_time)).replace("{$time_zone}", self.time_zone) - self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) - self.sql = self.sql.replace("{$time_filter}", time_filter) - self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) - - # query data from clickhouse database - try: - ipvanishvpn_servername_df = pd.DataFrame(self.client.execute(self.sql)) - finally: - self.client.disconnect() - - if ipvanishvpn_servername_df.empty: - self.logger.info('No ipvanishvpn server name found from session records') - return [] - ipvanishvpn_servername_list = ipvanishvpn_servername_df[0].drop_duplicates().tolist() - self.logger.info('Query ipvanishvpn server name from session records successfully. {} items found' - .format(len(ipvanishvpn_servername_list))) - - return ipvanishvpn_servername_list diff --git a/detection/vpnservices/ivacyvpn.py b/detection/vpnservices/ivacyvpn.py new file mode 100644 index 0000000..805ade1 --- /dev/null +++ b/detection/vpnservices/ivacyvpn.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2024/1/31 14:30 +# @author : yinjinagyi +# @File : ivacyvpn.py.py +# @Function: +import datetime + +import pandas as pd + +from tool.Functions import check_internet +from tool.MariadbTool import MariadbUtil +from vpn_detector import VpnDetector, ServerGroup + + +class Ivacyvpn(VpnDetector): + """ + + This class is used to detect ivacyvpn server ip and server name + """ + + def __init__(self, start_time, end_time): + super().__init__(start_time, end_time) + self.plugin_config = self.load_config()['ivacyvpn'] + self.vpn_service_name = self.plugin_config['vpn_service_name'] + self.plugin_name = self.plugin_config['plugin_name'] + self.plugin_id = self.plugin_config['plugin_id'] + self.confidence = self.plugin_config['confidence'] + self.start_time = start_time + self.end_time = end_time + + def find_server(self): + """ + Get ivacyvpn server ip and server name from clickhouse database + :return: ivacyvpn server ip list and server name list + """ + result_group = [] + + # start finding ivacyvpn server name + ivacyvpn_detector = IvacyvpnServername(self.start_time, self.end_time) + result_group.extend(ivacyvpn_detector.find_server()) + + # start finding ivacyvpn server ip + ivacyvpn_detector = IvacyvpnServerip() + result_group.extend(ivacyvpn_detector.find_server()) + + return result_group + + + +class IvacyvpnServerip(VpnDetector): + """ + + This class is used to detect ivacyvpn server ip + """ + + def __init__(self): + super().__init__('', '') + self.plugin_config = self.load_config()['ivacyvpn'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['ip']['object_type'] + # 开始时间为当前的整点时间 + self.start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:00:00") + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + + self.kb_sql = self.plugin_config['ip']['kb_sql'] + self.kb_dbname = self.config['knowledgebase']['db_name'] + self.kb_table_name = self.config['knowledgebase']['domain_library_name'] + + self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], + self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), + self.config['mariadb']['db_name']) + self.mariadb_dbname = self.config['mariadb']['db_name'] + self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] + self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] + + + + + def find_server(self): + """ + Find ivacyvpn server ip by resolving ivacyvpn server name + :return: server ip list + """ + self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) + + server_name_list = [] + server_ip_list = [] + + try: + query_result = self.mariadb.query_sql(self.kb_sql) + finally: + self.mariadb.close() + + if query_result: + for row in query_result: + server_name_list.append(row[0]) + + # add dc-xxx.pointtoserver.com to server_name_list + server_name_list.extend([f"dc-{str(index)}.pointtoserver.com" for index in range(1000)]) + server_name_list = list(set(server_name_list)) + + if check_internet(): + server_ip_list = self.resolve_dns_for_domain_list(server_name_list) + if server_ip_list: + server_ip_list = list(set(server_ip_list)) + self.logger.info( + '[{}] - Get {} server ip by resolving server name successfully.'.format(self.plugin_name, + len(server_ip_list))) + else: + self.logger.info("[{}] - No ivacyvpn server name found from knowledge database.".format(self.plugin_name)) + else: + self.logger.info("[{}] - No internet connection, skip dns resolve.".format(self.plugin_name)) + + return [ServerGroup(self.object_type, server_ip_list, self.output_file_name)] + + +class IvacyvpnServername(VpnDetector): + """ + + This class is used to detect ivacyvpn server name + """ + + def __init__(self, start_time, end_time): + super().__init__(start_time, end_time) + self.plugin_config = self.load_config()['ivacyvpn'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['domain']['object_type'] + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + self.start_time = start_time + self.end_time = end_time + + self.sql = self.plugin_config['domain']['sql'] + + def find_server(self): + """ + Get ivacyvpn server name from clickhouse database + :return: ivacyvpn server name list + """ + self.logger.info('[{}] - Start to query ivacyvpn server name from session record'.format(self.plugin_name)) + + # construct query sql + TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', + self.config['common'][ + 'recv_time_columnname']) + time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( + self.end_time)).replace("{$time_zone}", self.time_zone) + self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) + self.sql = self.sql.replace("{$time_filter}", time_filter) + self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_config['plugin_name'], self.sql)) + + # query data from clickhouse database + try: + ivacyvpn_servername_df = pd.DataFrame(self.client.execute(self.sql)) + finally: + self.client.disconnect() + + if ivacyvpn_servername_df.empty: + self.logger.info('[{}] - No ivacyvpn server name found from session records'.format(self.plugin_name)) + return [] + ivacyvpn_servername_list = ivacyvpn_servername_df[0].drop_duplicates().tolist() + self.logger.info('[{}] - Query ivacyvpn server name from session records successfully. {} items found' + .format(self.plugin_name, len(ivacyvpn_servername_list))) + + return [ServerGroup(self.object_type, ivacyvpn_servername_list, self.output_file_name)] diff --git a/detection/vpnservices/ivacyvpn_serverip.py b/detection/vpnservices/ivacyvpn_serverip.py deleted file mode 100644 index 69b21a0..0000000 --- a/detection/vpnservices/ivacyvpn_serverip.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2024/1/17 15:25 -# @author : yinjinagyi -# @File : ivacyvpn_serverip.py -# @Function: - -import sys - -sys.path.append('..') -from tool.Functions import check_internet -from tool.MariadbTool import MariadbUtil -from vpn_detector import VpnDetector - -class IvacyvpnServerip(VpnDetector): - """ - - This class is used to detect ivacyvpn server ip - """ - - def __init__(self, start_time, end_time): - super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['ivacyvpn_serverip'] - self.plugin_id = self.plugin_config['plugin_id'] - self.plugin_name = self.plugin_config['plugin_name'] - self.object_type = self.plugin_config['object_type'] - self.vpn_service_name = self.plugin_config['vpn_service_name'] - self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' - self.start_time = start_time - self.end_time = end_time - - self.kb_sql = self.plugin_config['kb_sql'] - self.kb_dbname = self.config['knowledgebase']['db_name'] - self.kb_table_name = self.config['knowledgebase']['domain_library_name'] - - self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], - self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), - self.config['mariadb']['db_name']) - - self.mariadb_dbname = self.config['mariadb']['db_name'] - self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] - self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] - - - - - def find_server(self): - """ - Find ivacyvpn server ip by resolving ivacyvpn server name - :return: server ip list - """ - self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) - - server_name_list = [] - server_ip_list = [] - - try: - query_result = self.mariadb.query_sql(self.kb_sql) - finally: - self.mariadb.close() - - if query_result: - for row in query_result: - server_name_list.append(row[0]) - - # add dc-xxx.pointtoserver.com to server_name_list - server_name_list.extend([f"dc-{str(index)}.pointtoserver.com" for index in range(1000)]) - server_name_list = list(set(server_name_list)) - - if check_internet(): - server_ip_list = self.resolve_dns_for_domain_list(server_name_list) - if server_ip_list: - server_ip_list = list(set(server_ip_list)) - else: - self.logger.info("No ivacyvpn server name found from knowledge database.") - else: - self.logger.info("No internet connection, skip dns resolve.") - return server_ip_list - - - - - diff --git a/detection/vpnservices/ivacyvpn_servername.py b/detection/vpnservices/ivacyvpn_servername.py deleted file mode 100644 index 896f291..0000000 --- a/detection/vpnservices/ivacyvpn_servername.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2024/1/17 15:22 -# @author : yinjinagyi -# @File : ivacyvpn_servername.py -# @Function: - -import sys -sys.path.append('..') -from vpn_detector import VpnDetector -import pandas as pd - -class IvacyvpnServername(VpnDetector): - """ - - This class is used to detect ivacyvpn server name - """ - - def __init__(self, start_time, end_time): - super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['ivacyvpn_servername'] - self.plugin_id = self.plugin_config['plugin_id'] - self.plugin_name = self.plugin_config['plugin_name'] - self.object_type = self.plugin_config['object_type'] - self.vpn_service_name = self.plugin_config['vpn_service_name'] - self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' - self.start_time = start_time - self.end_time = end_time - - self.sql = self.plugin_config['sql'] - - def find_server(self): - """ - Get ivacyvpn server name from clickhouse database - :return: ivacyvpn server name list - """ - self.logger.info('Start to query ivacyvpn server name from session record') - - # construct query sql - TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', self.config['common']['recv_time_columnname']) - time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( - self.end_time)).replace("{$time_zone}", self.time_zone) - self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) - self.sql = self.sql.replace("{$time_filter}", time_filter) - self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) - - # query data from clickhouse database - try: - ivacyvpn_servername_df = pd.DataFrame(self.client.execute(self.sql)) - finally: - self.client.disconnect() - - if ivacyvpn_servername_df.empty: - self.logger.info('No ivacyvpn server name found from session records') - return [] - ivacyvpn_servername_list = ivacyvpn_servername_df[0].drop_duplicates().tolist() - self.logger.info('Query ivacyvpn server name from session records successfully. {} items found' - .format(len(ivacyvpn_servername_list))) - return ivacyvpn_servername_list
\ No newline at end of file diff --git a/detection/vpnservices/protonvpn_serverip.py b/detection/vpnservices/protonvpn.py index 1f82dae..b3dac90 100644 --- a/detection/vpnservices/protonvpn_serverip.py +++ b/detection/vpnservices/protonvpn.py @@ -2,28 +2,27 @@ # -*- coding: utf-8 -*- # @Time : 2024/1/29 18:49 # @author : yinjinagyi -# @File : protonvpn_serverip.py +# @File : protonvpn.py # @Function: import pandas as pd -from vpn_detector import VpnDetector +from vpn_detector import VpnDetector, ServerGroup -class ProtonvpnServerip(VpnDetector): +class Protonvpn(VpnDetector): """ This class is used to detect protonvpn server ip """ def __init__(self, start_time, end_time): super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['protonvpn_serverip'] + self.plugin_config = self.load_config()['protonvpn'] self.plugin_id = self.plugin_config['plugin_id'] self.plugin_name = self.plugin_config['plugin_name'] self.object_type = self.plugin_config['object_type'] self.vpn_service_name = self.plugin_config['vpn_service_name'] self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[ - :13] + '.csv' + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' self.start_time = start_time self.end_time = end_time @@ -34,7 +33,7 @@ class ProtonvpnServerip(VpnDetector): Get protonvpn server ip from clickhouse database :return: protonvpn server ip list """ - self.logger.info('Start to query protonvpn server ip from session record') + self.logger.info('[{}] - Start to query protonvpn server ip from session record'.format(self.plugin_name)) # construct query sql TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', @@ -45,7 +44,7 @@ class ProtonvpnServerip(VpnDetector): self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) self.sql = self.sql.replace("{$time_filter}", time_filter) - self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) + self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_name, self.sql)) # query data from clickhouse database try: @@ -54,10 +53,10 @@ class ProtonvpnServerip(VpnDetector): self.client.disconnect() if protonvpn_serverip_df.empty: - self.logger.info('No protonvpn server ip found from session record') + self.logger.info('[{}] - No protonvpn server ip found from session record'.format(self.plugin_name)) return [] protonvpn_serverip_list = protonvpn_serverip_df[0].drop_duplicates().tolist() - self.logger.info('Query protonvpn server ip from session records successfully. {} items found' - .format(len(protonvpn_serverip_list))) + self.logger.info('[{}] - Query protonvpn server ip from session records successfully. {} items found' + .format(self.plugin_name, len(protonvpn_serverip_list))) - return protonvpn_serverip_list + return [ServerGroup(object_type='ip', server_list=protonvpn_serverip_list, output_file_name=self.output_file_name)] diff --git a/detection/vpnservices/turbovpn_serverip.py b/detection/vpnservices/turbovpn.py index 1546985..2c5ab87 100644 --- a/detection/vpnservices/turbovpn_serverip.py +++ b/detection/vpnservices/turbovpn.py @@ -2,26 +2,28 @@ # -*- coding: utf-8 -*- # @Time : 2024/1/17 18:09 # @author : yinjinagyi -# @File : turbovpn_serverip.py +# @File : turbovpn.py # @Function: import pandas as pd -from vpn_detector import VpnDetector +from vpn_detector import VpnDetector, ServerGroup -class TurbovpnServerip(VpnDetector): + +class Turbovpn(VpnDetector): """ This class is used to detect turbovpn server ip """ def __init__(self, start_time, end_time): super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['turbovpn_serverip'] + self.plugin_config = self.load_config()['turbovpn'] self.plugin_id = self.plugin_config['plugin_id'] self.plugin_name = self.plugin_config['plugin_name'] self.object_type = self.plugin_config['object_type'] self.vpn_service_name = self.plugin_config['vpn_service_name'] self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ','_').replace(':', '')[:13] + '.csv' + self.start_time = start_time self.end_time = end_time @@ -44,7 +46,7 @@ class TurbovpnServerip(VpnDetector): self.sql = self.sql.replace("{$security_table_name}", self.plugin_config['security_table_name'])\ .replace("{$security_policy_id}", str(self.plugin_config['security_policy_id'])) - self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) + self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_name, self.sql)) # query data from clickhouse database try: @@ -53,9 +55,12 @@ class TurbovpnServerip(VpnDetector): self.client.disconnect() if turbovpn_serverip_df.empty: - self.logger.info('No turbovpn server ip found from session record') + self.logger.error('[{}] - No turbovpn server ip found from security event. Policy id: {}.'.format(self.plugin_name, self.plugin_config['security_policy_id'])) return [] turbovpn_serverip_list = turbovpn_serverip_df[0].drop_duplicates().tolist() - self.logger.info('Query turbovpn server ip from session records successfully. {} items found' - .format(len(turbovpn_serverip_list))) - return turbovpn_serverip_list + self.logger.info('[{}] - Query turbovpn server ip from clickhouse database successfully. {} items found' + .format(self.plugin_name, len(turbovpn_serverip_list))) + + return [ + ServerGroup(object_type='ip', server_list=turbovpn_serverip_list, output_file_name=self.output_file_name)] + diff --git a/detection/vpnservices/vpnunlimited_serverip.py b/detection/vpnservices/vpnunlimited.py index 4aea727..f168af1 100644 --- a/detection/vpnservices/vpnunlimited_serverip.py +++ b/detection/vpnservices/vpnunlimited.py @@ -2,13 +2,13 @@ # -*- coding: utf-8 -*- # @Time : 2024/1/23 10:39 # @author : yinjinagyi -# @File : vpnunlimited_serverip.py +# @File : vpnunlimited.py # @Function: -from vpn_detector import VpnDetector +from vpn_detector import VpnDetector, ServerGroup import pandas as pd -class VpnunlimitedServerip(VpnDetector): +class Vpnunlimited(VpnDetector): """ This class is used to detect vpnunlimited server ip @@ -16,13 +16,15 @@ class VpnunlimitedServerip(VpnDetector): def __init__(self, start_time, end_time): super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['vpnunlimited_serverip'] + self.plugin_config = self.load_config()['vpnunlimited'] self.plugin_id = self.plugin_config['plugin_id'] self.plugin_name = self.plugin_config['plugin_name'] self.object_type = self.plugin_config['object_type'] self.vpn_service_name = self.plugin_config['vpn_service_name'] self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ','_').replace(':', '')[:13] + '.csv' + + self.start_time = start_time self.end_time = end_time @@ -34,7 +36,7 @@ class VpnunlimitedServerip(VpnDetector): Get vpnunlimited server ip from session records :return: vpnunlimited server ip list """ - self.logger.info('Start to query vpnunlimited server ip from session records') + self.logger.info('[{}] - Start to query server ip from session records'.format(self.plugin_name)) # construct query sql TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', self.config['common']['recv_time_columnname']) @@ -44,7 +46,7 @@ class VpnunlimitedServerip(VpnDetector): self.sql = self.sql.replace("{$time_filter}", time_filter) self.sql = self.sql.replace("{$domain_list}", ','.join(self.domains)) - self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) + self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_name, self.sql)) # query data from clickhouse database try: @@ -53,10 +55,10 @@ class VpnunlimitedServerip(VpnDetector): self.client.disconnect() if vpnunlimited_serverip_df.empty: - self.logger.info('No vpnunlimited server ip found from session records') + self.logger.info('[{}] - No server ip found from session records'.format(self.plugin_name)) return [] vpnunlimited_serverip_list = vpnunlimited_serverip_df[0].drop_duplicates().tolist() - self.logger.info('Query vpnunlimited server ip from session records successfully. {} items found' - .format(len(vpnunlimited_serverip_list))) + self.logger.info('[{}] - Query server ip from clickhouse database successfully. {} items found' + .format(self.plugin_name, len(vpnunlimited_serverip_list))) - return vpnunlimited_serverip_list
\ No newline at end of file + return [ServerGroup(object_type='ip', server_list=vpnunlimited_serverip_list, output_file_name=self.output_file_name)] diff --git a/detection/vpnservices/windscribevpn.py b/detection/vpnservices/windscribevpn.py new file mode 100644 index 0000000..a45c4d9 --- /dev/null +++ b/detection/vpnservices/windscribevpn.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2024/1/31 15:23 +# @author : yinjinagyi +# @File : windscribevpn.py +# @Function: +import datetime +import re + +import pandas as pd +from statsmodels.datasets import check_internet + +from tool.MariadbTool import MariadbUtil +from vpn_detector import VpnDetector, ServerGroup + + +class Windscribevpn(VpnDetector): + """ + + This class is used to detect windscribevpn server ip and server name + """ + + def __init__(self, start_time, end_time): + super().__init__(start_time, end_time) + self.plugin_config = self.load_config()['windscribevpn'] + self.vpn_service_name = self.plugin_config['vpn_service_name'] + self.plugin_name = self.plugin_config['plugin_name'] + self.plugin_id = self.plugin_config['plugin_id'] + self.confidence = self.plugin_config['confidence'] + self.start_time = start_time + self.end_time = end_time + + def find_server(self): + """ + Get windscribevpn server ip and server name from clickhouse database + :return: windscribevpn server ip list and server name list + """ + result_group = [] + + # start finding windscribevpn server name + windscribevpn_detector = WindscribevpnServername(self.start_time, self.end_time) + result_group.extend(windscribevpn_detector.find_server()) + + # start finding windscribevpn server ip + windscribevpn_detector = WindscribevpnServerip() + result_group.extend(windscribevpn_detector.find_server()) + + return result_group + + +class WindscribevpnServerip(VpnDetector): + """ + + This class is used to detect windscribevpn server ip + """ + def __init__(self): + super().__init__('', '') + self.plugin_config = self.load_config()['windscribevpn'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['ip']['object_type'] + # 开始时间为当前的整点时间 + self.start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:00:00") + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + + self.kb_sql = self.plugin_config['ip']['kb_sql'] + self.kb_dbname = self.config['knowledgebase']['db_name'] + self.kb_table_name = self.config['knowledgebase']['domain_library_name'] + + self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], + self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), + self.config['mariadb']['db_name']) + self.mariadb_dbname = self.config['mariadb']['db_name'] + self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] + self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] + + + def find_more_servernames(self, server_name_list): + """ + Find more server name from observed windscribevpn server name list + :return: server name list + """ + prefix_list = [] + expanded_server_names = [] + + pattern = re.compile(r'\D+(\d+)\.\w+\.\w+') + for server_name in server_name_list: + domain = server_name.strip() + match = pattern.match(domain) + if match: + numeric_part = match.group(1) + domain_pattern = re.sub(numeric_part, '{index}', domain) + prefix_list.append(domain_pattern) + else: + continue + + prefix_list = set(prefix_list) + + for domain_prefix in prefix_list: + domain_list = [re.sub(r'{index}', str(index).zfill(3), domain_prefix) for index in range(1000)] + expanded_server_names.extend(domain_list) + return expanded_server_names + + + def find_server(self): + """ + Get windscribevpn server ip by resolving windscribevpn server name + :return: windscribevpn server ip list + """ + self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace( + "{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) + + servername_list = [] + resolved_ip_list = [] + try: + query_result = self.mariadb.query_sql(self.kb_sql) + finally: + self.mariadb.close() + + if query_result: + servername_list = [i[0] for i in query_result] + + # 判断是否能够访问外网，如果能够访问外网，则从外网获取windscribevpn_servername_list的域名解析地址 + if check_internet(): + servername_list = self.find_more_servernames(servername_list) + if len(servername_list) > 0: + resolved_ip_list = self.resolve_dns_for_domain_list(servername_list) + self.logger.info( + '[{}] - Get {} server ip by resolving server name successfully.'.format( + self.plugin_name, len(resolved_ip_list))) + else: + self.logger.info( + '[{}] - No windscribevpn server name found from knowledge database.'.format(self.plugin_name)) + else: + self.logger.info('[{}] - No internet connection, skip dns resolve.'.format(self.plugin_name)) + + return [ServerGroup(self.object_type, resolved_ip_list, self.output_file_name)] + + + +class WindscribevpnServername(VpnDetector): + """ + + This class is used to detect windscribevpn server name + """ + + def __init__(self, start_time, end_time): + super().__init__(start_time, end_time) + self.plugin_config = self.load_config()['windscribevpn'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['domain']['object_type'] + self.output_file_name = self.plugin_name + '-' + self.object_type + '_' + str(self.start_time).replace(' ', + '_').replace( + ':', '')[:13] + '.csv' + self.start_time = start_time + self.end_time = end_time + + self.sql = self.plugin_config['domain']['sql'] + self.domains = ["'" + i.strip() + "'" for i in self.plugin_config['domain']['domains'].split(',')] + + def find_server(self): + """ + Get windscribevpn server name from session records + :return: windscribevpn server name list + """ + self.logger.info('[{}] - Start to query windscribevpn server name from session record'.format(self.plugin_name)) + + # construct query sql + TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', + self.config['common'][ + 'recv_time_columnname']) + time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( + self.end_time)).replace("{$time_zone}", self.time_zone) + self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) + self.sql = self.sql.replace("{$time_filter}", time_filter) + self.sql = self.sql.replace("{$domain_list}", ','.join(self.domains)) + self.logger.info("[{}] - Sql for {}: {}".format(self.plugin_name, self.plugin_config['plugin_name'], self.sql)) + + # query data from clickhouse database + try: + windscribevpn_servername_df = pd.DataFrame(self.client.execute(self.sql)) + finally: + self.client.disconnect() + + if windscribevpn_servername_df.empty: + self.logger.info('[{}] - No windscribevpn server name found from session records'.format(self.plugin_name)) + return [] + windscribevpn_servername_list = windscribevpn_servername_df[0].drop_duplicates().tolist() + self.logger.info('[{}] - Query windscribevpn server name from session records successfully. {} items found' + .format(self.plugin_name, len(windscribevpn_servername_list))) + + return [ServerGroup(self.object_type, windscribevpn_servername_list, self.output_file_name)] + + +def extract_pattern(domain): + pattern = re.compile(r'\D+(\d+)\.\w+\.\w+') + + match = pattern.match(domain) + if match: + numeric_part = match.group(1) + domain_pattern = re.sub(numeric_part, '{index}', domain) + return domain_pattern + else: + return
\ No newline at end of file diff --git a/detection/vpnservices/windscribevpn_serverip.py b/detection/vpnservices/windscribevpn_serverip.py deleted file mode 100644 index 2d21c06..0000000 --- a/detection/vpnservices/windscribevpn_serverip.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2024/1/24 15:18 -# @author : yinjinagyi -# @File : windscribevpn_serverip.py -# @Function: -import re - -import sys -sys.path.append('..') -from statsmodels.datasets import check_internet - -from tool.MariadbTool import MariadbUtil -from vpn_detector import VpnDetector - - -class WindscribevpnServerip(VpnDetector): - """ - - This class is used to detect windscribevpn server ip - """ - def __init__(self): - super().__init__('', '') - self.plugin_config = self.load_config()['windscribevpn_serverip'] - self.plugin_id = self.plugin_config['plugin_id'] - self.plugin_name = self.plugin_config['plugin_name'] - self.object_type = self.plugin_config['object_type'] - self.vpn_service_name = self.plugin_config['vpn_service_name'] - self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' - - self.kb_sql = self.plugin_config['kb_sql'] - self.kb_dbname = self.config['knowledgebase']['db_name'] - self.kb_table_name = self.config['knowledgebase']['domain_library_name'] - - self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], - self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), - self.config['mariadb']['db_name']) - self.mariadb_dbname = self.config['mariadb']['db_name'] - self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] - self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] - - - def find_more_servernames(self, server_name_list): - """ - Find more server name from observed windscribe server name list - :return: server name list - """ - prefix_list = [] - expanded_server_names = [] - - pattern = re.compile(r'\D+(\d+)\.\w+\.\w+') - for server_name in server_name_list: - domain = server_name.strip() - match = pattern.match(domain) - if match: - numeric_part = match.group(1) - domain_pattern = re.sub(numeric_part, '{index}', domain) - prefix_list.append(domain_pattern) - else: - continue - - prefix_list = set(prefix_list) - - for domain_prefix in prefix_list: - domain_list = [re.sub(r'{index}', str(index).zfill(3), domain_prefix) for index in range(1000)] - expanded_server_names.extend(domain_list) - return expanded_server_names - - - def find_server(self): - """ - Get windscribevpn server ip by resolving windscribevpn server name - :return: windscribevpn server ip list - """ - self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) - - servername_list = [] - resolved_ip_list = [] - try: - query_result = self.mariadb.query_sql(self.kb_sql) - finally: - self.mariadb.close() - - if query_result: - servername_list = [i[0] for i in query_result] - - # 判断是否能够访问外网，如果能够访问外网，则从外网获取windscribe_servername_list的域名解析地址 - if check_internet(): - servername_list = self.find_more_servernames(servername_list) - if len(servername_list) > 0: - resolved_ip_list = self.resolve_dns_for_domain_list(servername_list) - else: - self.logger.info('No windscribe server name found from knowledge base') - else: - self.logger.info('No internet access, skip to resolve windscribe server name') - - return resolved_ip_list - - - -def extract_pattern(domain): - pattern = re.compile(r'\D+(\d+)\.\w+\.\w+') - - match = pattern.match(domain) - if match: - numeric_part = match.group(1) - domain_pattern = re.sub(numeric_part, '{index}', domain) - return domain_pattern - else: - return diff --git a/detection/vpnservices/windscribevpn_servername.py b/detection/vpnservices/windscribevpn_servername.py deleted file mode 100644 index fb90c5c..0000000 --- a/detection/vpnservices/windscribevpn_servername.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2024/1/24 15:08 -# @author : yinjinagyi -# @File : windscribevpn_servername.py -# @Function: - -from vpn_detector import VpnDetector -import pandas as pd - - -class WindscribevpnServername(VpnDetector): - """ - - This class is used to detect windscribevpn server name - """ - - def __init__(self, start_time, end_time): - super().__init__(start_time, end_time) - self.plugin_config = self.load_config()['windscribevpn_servername'] - self.plugin_id = self.plugin_config['plugin_id'] - self.plugin_name = self.plugin_config['plugin_name'] - self.object_type = self.plugin_config['object_type'] - self.vpn_service_name = self.plugin_config['vpn_service_name'] - self.confidence = self.plugin_config['confidence'] - self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[ - :13] + '.csv' - self.start_time = start_time - self.end_time = end_time - - self.sql = self.plugin_config['sql'] - self.domains = ["'" + i.strip() + "'" for i in self.plugin_config['domains'].split(',')] - - def find_server(self): - """ - Get windscribevpn server name from session records - :return: windscribevpn server name list - """ - self.logger.info('Start to query windscribevpn server name from session records') - - # construct query sql - TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', - self.config['common'][ - 'recv_time_columnname']) - time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( - self.end_time)).replace("{$time_zone}", self.time_zone) - self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) - self.sql = self.sql.replace("{$time_filter}", time_filter) - self.sql = self.sql.replace("{$domain_list}", ','.join(self.domains)) - - self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) - - # query data from clickhouse database - try: - windscribevpn_servername_df = pd.DataFrame(self.client.execute(self.sql)) - finally: - self.client.disconnect() - - if windscribevpn_servername_df.empty: - self.logger.info('No windscribevpn server name found from session records') - return [] - windscribevpn_servername_list = windscribevpn_servername_df[0].drop_duplicates().tolist() - self.logger.info('Query windscribevpn server name from session records successfully. {} items found' - .format(len(windscribevpn_servername_list))) - return windscribevpn_servername_list |