From 1ca6d08ccdee730fae3014f6ad73e5d6c6ee9f5b Mon Sep 17 00:00:00 2001 From: 尹姜谊 Date: Wed, 24 Jan 2024 17:33:22 +0800 Subject: Add: windscribevpn plugin --- config23.10.yaml | 19 ++++ config24.01.yaml | 20 +++- detection/tool/Config.py | 4 +- detection/vpnservices/cyberghostvpn_serverip.py | 6 +- detection/vpnservices/ipvanishvpn_serverip.py | 7 +- detection/vpnservices/ivacyvpn_serverip.py | 5 +- detection/vpnservices/vpnunlimited_serverip.py | 8 +- detection/vpnservices/windscribevpn_serverip.py | 109 ++++++++++++++++++++++ detection/vpnservices/windscribevpn_servername.py | 65 +++++++++++++ test/test_plugins.py | 41 ++++++++ 10 files changed, 266 insertions(+), 18 deletions(-) create mode 100644 detection/vpnservices/windscribevpn_serverip.py create mode 100644 detection/vpnservices/windscribevpn_servername.py diff --git a/config23.10.yaml b/config23.10.yaml index 422ce11..ea2445c 100644 --- a/config23.10.yaml +++ b/config23.10.yaml @@ -139,3 +139,22 @@ vpnunlimited_serverip: confidence: confirmed sql: SELECT DISTINCT common_server_ip FROM {$db_name}.{$table_name} WHERE {$time_filter} AND common_server_domain in ({$domain_list}) domains: hurriwhilealivo.club, comcatches.live, cyphyl.com, chinacitybit.click, valarre.com, puppyfood.info, securestartup.business, beansandchips.com, zigzagwand.art, wifimeshnet.cc, atomicspike.art, fastwaterblog.com, aspheric-zombies.club, godzillo.link, cyberroast.shop, seligmania-online.com, easy-2fa.us, ikitoshi.cc, webcitynews.com, prebreeze.club, blackbettyclothing.com, cyberanalytics.link, musicinst.link, adsoasis.xyz, holidayphoto.xyz, graphlist.dev, nohumguitar.com, coffeedaybreak.com, thewalruss.net, learnjapanfilms.cc, ezhyperlix.xyz, statsnet.group, hockeybet.org, fastblazingpix.com, zapp-a-weasel.live + + +windscribevpn_servername: + vpn_service_name: windscribevpn + plugin_id: 12 + plugin_name: windscribevpn_servername + object_type: domain + confidence: confirmed + sql: SELECT DISTINCT common_server_fqdn FROM {$db_name}.{$table_name} WHERE {$time_filter} and common_server_domain in ({$domain_list}) and common_server_fqdn like '%-%' ORDER BY common_server_fqdn ASC + domains: whiskergalaxy.com, totallyacdn.com + + +windscribevpn_serverip: + vpn_service_name: windscribevpn + plugin_id: 13 + plugin_name: windscribevpn_serverip + object_type: ip + confidence: confirmed + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'windscribevpn' diff --git a/config24.01.yaml b/config24.01.yaml index b342454..666f70f 100644 --- a/config24.01.yaml +++ b/config24.01.yaml @@ -110,7 +110,6 @@ ivacyvpn_servername: sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.pointtoserver.com' - ivacyvpn_serverip: vpn_service_name: ivacyvpn plugin_id: 9 @@ -139,3 +138,22 @@ vpnunlimited_serverip: confidence: confirmed sql: SELECT DISTINCT server_ip FROM {$db_name}.{$table_name} WHERE {$time_filter} AND server_domain in ({$domain_list}) domains: hurriwhilealivo.club, comcatches.live, cyphyl.com, chinacitybit.click, valarre.com, puppyfood.info, securestartup.business, beansandchips.com, zigzagwand.art, wifimeshnet.cc, atomicspike.art, fastwaterblog.com, aspheric-zombies.club, godzillo.link, cyberroast.shop, seligmania-online.com, easy-2fa.us, ikitoshi.cc, webcitynews.com, prebreeze.club, blackbettyclothing.com, cyberanalytics.link, musicinst.link, adsoasis.xyz, holidayphoto.xyz, graphlist.dev, nohumguitar.com, coffeedaybreak.com, thewalruss.net, learnjapanfilms.cc, ezhyperlix.xyz, statsnet.group, hockeybet.org, fastblazingpix.com, zapp-a-weasel.live + + +windscribevpn_servername: + vpn_service_name: windscribevpn + plugin_id: 12 + plugin_name: windscribevpn_servername + object_type: domain + confidence: confirmed + sql: SELECT DISTINCT server_fqdn FROM {$db_name}.{$table_name} WHERE {$time_filter} and server_domain in ({$domain_list}) and server_fqdn like '%-%' ORDER BY server_fqdn ASC + domains: whiskergalaxy.com, totallyacdn.com + + +windscribevpn_serverip: + vpn_service_name: windscribevpn + plugin_id: 13 + plugin_name: windscribevpn_serverip + object_type: ip + confidence: confirmed + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'windscribevpn' \ No newline at end of file diff --git a/detection/tool/Config.py b/detection/tool/Config.py index ca23e5f..a5d2b19 100644 --- a/detection/tool/Config.py +++ b/detection/tool/Config.py @@ -9,12 +9,12 @@ import yaml import os import sys -import LoggingTool sys.path.append('..') +from tool.LoggingTool import Logger from tool.Functions import get_project_path -logger = LoggingTool.Logger().getLogger() +logger = Logger().getLogger() class Config: diff --git a/detection/vpnservices/cyberghostvpn_serverip.py b/detection/vpnservices/cyberghostvpn_serverip.py index 7401587..4505a95 100644 --- a/detection/vpnservices/cyberghostvpn_serverip.py +++ b/detection/vpnservices/cyberghostvpn_serverip.py @@ -16,8 +16,8 @@ class CyberghostvpnServerip(VpnDetector): This class is used to detect cyberghostvpn server ip """ - def __init__(self, start_time, end_time): - super().__init__(start_time, end_time) + def __init__(self): + super().__init__('', '') self.plugin_config = self.load_config()['cyberghostvpn_serverip'] self.plugin_id = self.plugin_config['plugin_id'] self.plugin_name = self.plugin_config['plugin_name'] @@ -25,8 +25,6 @@ class CyberghostvpnServerip(VpnDetector): self.vpn_service_name = self.plugin_config['vpn_service_name'] self.confidence = self.plugin_config['confidence'] self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' - self.start_time = start_time - self.end_time = end_time self.kb_sql = self.plugin_config['kb_sql'] self.kb_dbname = self.config['knowledgebase']['db_name'] diff --git a/detection/vpnservices/ipvanishvpn_serverip.py b/detection/vpnservices/ipvanishvpn_serverip.py index 3229971..9a26407 100644 --- a/detection/vpnservices/ipvanishvpn_serverip.py +++ b/detection/vpnservices/ipvanishvpn_serverip.py @@ -15,8 +15,8 @@ class IpvanishvpnServerip(VpnDetector): This class is used to detect ipvanishvpn server ip """ - def __init__(self, start_time, end_time): - super().__init__(start_time, end_time) + def __init__(self): + super().__init__('', '') self.plugin_config = self.load_config()['ipvanishvpn_serverip'] self.plugin_id = self.plugin_config['plugin_id'] self.plugin_name = self.plugin_config['plugin_name'] @@ -24,8 +24,7 @@ class IpvanishvpnServerip(VpnDetector): self.vpn_service_name = self.plugin_config['vpn_service_name'] self.confidence = self.plugin_config['confidence'] self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' - self.start_time = start_time - self.end_time = end_time + self.kb_sql = self.plugin_config['kb_sql'] self.kb_dbname = self.config['knowledgebase']['db_name'] diff --git a/detection/vpnservices/ivacyvpn_serverip.py b/detection/vpnservices/ivacyvpn_serverip.py index 0a65965..69b21a0 100644 --- a/detection/vpnservices/ivacyvpn_serverip.py +++ b/detection/vpnservices/ivacyvpn_serverip.py @@ -7,10 +7,9 @@ import sys -from Functions import check_internet -from MariadbTool import MariadbUtil - sys.path.append('..') +from tool.Functions import check_internet +from tool.MariadbTool import MariadbUtil from vpn_detector import VpnDetector class IvacyvpnServerip(VpnDetector): diff --git a/detection/vpnservices/vpnunlimited_serverip.py b/detection/vpnservices/vpnunlimited_serverip.py index 97221b8..4aea727 100644 --- a/detection/vpnservices/vpnunlimited_serverip.py +++ b/detection/vpnservices/vpnunlimited_serverip.py @@ -27,11 +27,11 @@ class VpnunlimitedServerip(VpnDetector): self.end_time = end_time self.sql = self.plugin_config['sql'] - self.masquerede_domains = ["'"+i.strip()+"'" for i in self.plugin_config['domains'].split(',')] + self.domains = ["'" + i.strip() + "'" for i in self.plugin_config['domains'].split(',')] def find_server(self): """ - Get vpnunlimited server ip from clickhouse database + Get vpnunlimited server ip from session records :return: vpnunlimited server ip list """ self.logger.info('Start to query vpnunlimited server ip from session records') @@ -42,7 +42,7 @@ class VpnunlimitedServerip(VpnDetector): self.end_time)).replace("{$time_zone}", self.time_zone) self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) self.sql = self.sql.replace("{$time_filter}", time_filter) - self.sql = self.sql.replace("{$domain_list}", ','.join(self.masquerede_domains)) + self.sql = self.sql.replace("{$domain_list}", ','.join(self.domains)) self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) @@ -56,7 +56,7 @@ class VpnunlimitedServerip(VpnDetector): self.logger.info('No vpnunlimited server ip found from session records') return [] vpnunlimited_serverip_list = vpnunlimited_serverip_df[0].drop_duplicates().tolist() - self.logger.info('Query vpnunlimited server ip from clickhouse database successfully. {} items found' + self.logger.info('Query vpnunlimited server ip from session records successfully. {} items found' .format(len(vpnunlimited_serverip_list))) return vpnunlimited_serverip_list \ No newline at end of file diff --git a/detection/vpnservices/windscribevpn_serverip.py b/detection/vpnservices/windscribevpn_serverip.py new file mode 100644 index 0000000..7ccb497 --- /dev/null +++ b/detection/vpnservices/windscribevpn_serverip.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2024/1/24 15:18 +# @author : yinjinagyi +# @File : windscribevpn_serverip.py +# @Function: +import re + +from statsmodels.datasets import check_internet + +from MariadbTool import MariadbUtil +from vpn_detector import VpnDetector + + +class WindscribevpnServerip(VpnDetector): + """ + + This class is used to detect windscribevpn server ip + """ + def __init__(self): + super().__init__('', '') + self.plugin_config = self.load_config()['windscribevpn_serverip'] + self.plugin_id = self.plugin_config['plugin_id'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['object_type'] + self.vpn_service_name = self.plugin_config['vpn_service_name'] + self.confidence = self.plugin_config['confidence'] + self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[:13] + '.csv' + + self.kb_sql = self.plugin_config['kb_sql'] + self.kb_dbname = self.config['knowledgebase']['db_name'] + self.kb_table_name = self.config['knowledgebase']['domain_library_name'] + + self.mariadb = MariadbUtil(self.config['mariadb']['host'], self.config['mariadb']['port'], + self.config['mariadb']['user'], str(self.config['mariadb']['pswd']), + self.config['mariadb']['db_name']) + self.mariadb_dbname = self.config['mariadb']['db_name'] + self.mariadb_ip_tb_name = self.config['mariadb']['ip_table_name'] + self.mariadb_domain_tb_name = self.config['mariadb']['domain_table_name'] + + + def find_more_servernames(self, server_name_list): + """ + Find more server name from observed windscribe server name list + :return: server name list + """ + prefix_list = [] + expanded_server_names = [] + + pattern = re.compile(r'\D+(\d+)\.\w+\.\w+') + for server_name in server_name_list: + domain = server_name.strip() + match = pattern.match(domain) + if match: + numeric_part = match.group(1) + domain_pattern = re.sub(numeric_part, '{index}', domain) + prefix_list.append(domain_pattern) + else: + continue + + prefix_list = set(prefix_list) + + for domain_prefix in prefix_list: + domain_list = [re.sub(r'{index}', str(index).zfill(3), domain_prefix) for index in range(1000)] + expanded_server_names.extend(domain_list) + return expanded_server_names + + + def find_server(self): + """ + Get windscribevpn server ip by resolving windscribevpn server name + :return: windscribevpn server ip list + """ + self.kb_sql = self.kb_sql.replace("{$mariadb_dbname}", self.mariadb_dbname).replace("{$mariadb_domain_tablename}", self.mariadb_domain_tb_name) + + servername_list = [] + resolved_ip_list = [] + try: + query_result = self.mariadb.query_sql(self.kb_sql) + finally: + self.mariadb.close() + + if query_result: + servername_list = [i[0] for i in query_result] + + # 判断是否能够访问外网，如果能够访问外网，则从外网获取windscribe_servername_list的域名解析地址 + if check_internet(): + servername_list = self.find_more_servernames(servername_list) + if len(servername_list) > 0: + resolved_ip_list = self.resolve_dns_for_domain_list(servername_list) + else: + self.logger.info('No windscribe server name found from knowledge base') + else: + self.logger.info('No internet access, skip to resolve windscribe server name') + + return resolved_ip_list + + + +def extract_pattern(domain): + pattern = re.compile(r'\D+(\d+)\.\w+\.\w+') + + match = pattern.match(domain) + if match: + numeric_part = match.group(1) + domain_pattern = re.sub(numeric_part, '{index}', domain) + return domain_pattern + else: + return diff --git a/detection/vpnservices/windscribevpn_servername.py b/detection/vpnservices/windscribevpn_servername.py new file mode 100644 index 0000000..fb90c5c --- /dev/null +++ b/detection/vpnservices/windscribevpn_servername.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Time : 2024/1/24 15:08 +# @author : yinjinagyi +# @File : windscribevpn_servername.py +# @Function: + +from vpn_detector import VpnDetector +import pandas as pd + + +class WindscribevpnServername(VpnDetector): + """ + + This class is used to detect windscribevpn server name + """ + + def __init__(self, start_time, end_time): + super().__init__(start_time, end_time) + self.plugin_config = self.load_config()['windscribevpn_servername'] + self.plugin_id = self.plugin_config['plugin_id'] + self.plugin_name = self.plugin_config['plugin_name'] + self.object_type = self.plugin_config['object_type'] + self.vpn_service_name = self.plugin_config['vpn_service_name'] + self.confidence = self.plugin_config['confidence'] + self.output_file_name = self.plugin_name + '_' + str(self.start_time).replace(' ', '_').replace(':', '')[ + :13] + '.csv' + self.start_time = start_time + self.end_time = end_time + + self.sql = self.plugin_config['sql'] + self.domains = ["'" + i.strip() + "'" for i in self.plugin_config['domains'].split(',')] + + def find_server(self): + """ + Get windscribevpn server name from session records + :return: windscribevpn server name list + """ + self.logger.info('Start to query windscribevpn server name from session records') + + # construct query sql + TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', + self.config['common'][ + 'recv_time_columnname']) + time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( + self.end_time)).replace("{$time_zone}", self.time_zone) + self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) + self.sql = self.sql.replace("{$time_filter}", time_filter) + self.sql = self.sql.replace("{$domain_list}", ','.join(self.domains)) + + self.logger.info("Sql for {}: {}".format(self.plugin_name, self.sql)) + + # query data from clickhouse database + try: + windscribevpn_servername_df = pd.DataFrame(self.client.execute(self.sql)) + finally: + self.client.disconnect() + + if windscribevpn_servername_df.empty: + self.logger.info('No windscribevpn server name found from session records') + return [] + windscribevpn_servername_list = windscribevpn_servername_df[0].drop_duplicates().tolist() + self.logger.info('Query windscribevpn server name from session records successfully. {} items found' + .format(len(windscribevpn_servername_list))) + return windscribevpn_servername_list diff --git a/test/test_plugins.py b/test/test_plugins.py index 1cc1f4c..835491a 100644 --- a/test/test_plugins.py +++ b/test/test_plugins.py @@ -19,6 +19,8 @@ from ivacyvpn_servername import IvacyvpnServername from detection.vpnservices.turbovpn_serverip import TurbovpnServerip from vpnunlimited_serverip import VpnunlimitedServerip +from windscribevpn_servername import WindscribevpnServername +from windscribevpn_serverip import WindscribevpnServerip class TestHotspotvpnServerip(unittest.TestCase): @@ -209,6 +211,45 @@ class TestVpnunlimitedServerip(unittest.TestCase): output_filename=self.vpnunlimited_detector.output_file_name, confidence=self.vpnunlimited_detector.confidence) +class TestWindscribevpnServername(unittest.TestCase): + + def setUp(self): + self.windscribevpn_detector = WindscribevpnServername(start_time='2024-01-16 00:00:00', + end_time='2024-01-19 00:00:00') + + def test_get_windscribevpn_servername(self): + self.result = self.windscribevpn_detector.find_server() + print(self.result) + + def test_save_to_knowledgebase(self): + self.windscribevpn_detector.save_to_knowledgebase(object_type='domain', + object_list=self.windscribevpn_detector.find_server(), + vpn_service_name=self.windscribevpn_detector.vpn_service_name, + plugin_id=self.windscribevpn_detector.plugin_id, + plugin_name=self.windscribevpn_detector.plugin_name, + output_filename=self.windscribevpn_detector.output_file_name, + confidence=self.windscribevpn_detector.confidence) + + + +class TestWindscribevpnServerip(unittest.TestCase): + + def setUp(self): + self.windscribevpn_detector = WindscribevpnServerip() + + def test_get_windscribevpn_serverip(self): + self.result = self.windscribevpn_detector.find_server() + print(self.result) + + def test_save_to_knowledgebase(self): + self.windscribevpn_detector.save_to_knowledgebase(object_type='ip', + object_list=self.windscribevpn_detector.find_server(), + vpn_service_name=self.windscribevpn_detector.vpn_service_name, + plugin_id=self.windscribevpn_detector.plugin_id, + plugin_name=self.windscribevpn_detector.plugin_name, + output_filename=self.windscribevpn_detector.output_file_name, + confidence=self.windscribevpn_detector.confidence) + if __name__ == '__main__': # # 运行指定测试用例 -- cgit v1.2.3