diff options
| -rw-r--r-- | config.yaml | 30 | ||||
| -rw-r--r-- | controller.py | 101 | ||||
| -rw-r--r-- | detection/vpnservices/hotspotvpn_serverip.py | 2 | ||||
| -rw-r--r-- | detection/vpnservices/ipvanishvpn_servername.py | 2 | ||||
| -rw-r--r-- | logs/run.log | 15 |
5 files changed, 31 insertions, 119 deletions
diff --git a/config.yaml b/config.yaml index 14c1731..2ec0a21 100644 --- a/config.yaml +++ b/config.yaml @@ -1,6 +1,8 @@ common: output_path: data/ time_zone: Asia/Shanghai + recv_time_columnname: common_recv_time + time_filter_pattern: (recv_time_columnname> toDateTime('{$start_time}', '{$time_zone}')) AND(recv_time_columnname <= toDateTime('{$end_time}', '{$time_zone}')) clickhouse: host: 192.168.40.194 @@ -58,4 +60,30 @@ ipvanishvpn_serverip: plugin_name: ipvanishvpn_serverip object_type: ip confidence: confirmed - kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'ipvanishvpn'
\ No newline at end of file + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'ipvanishvpn' + + +psiphon3vpn_serverip: + vpn_service_name: psiphon3vpn + plugin_id: 4 + plugin_name: psiphon3vpn_serverip + object_type: ip + confidence: + + +cyberghostvpn_servername: + vpn_service_name: cyberghostvpn + plugin_id: 5 + plugin_name: cyberghostvpn_servername + object_type: domain + confidence: confirmed + sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.nodes.gen4.ninja' + + +cyberghostvpn_serverip: + vpn_service_name: cyberghostvpn + plugin_id: 6 + plugin_name: cyberghostvpn_serverip + object_type: ip + confidence: confirmed + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'cyberghostvpn'
\ No newline at end of file diff --git a/controller.py b/controller.py deleted file mode 100644 index dc2d409..0000000 --- a/controller.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Time : 2023/3/18 20:02 -# @author : yinjinagyi -# @File : controller.py.py -# @Function: -import datetime -import sys - -import pytz -from codev.detection.ModelTraning import ModelTraining -from codev.tool import toolFunction, LoggingTool -from codev.detection.ModelDetection import ModelDetection -from codev.detection.Monitor import Monitor - -logger = LoggingTool.Logger().getLogger() - -if __name__ == '__main__': - - # 全局参数获取 - data = toolFunction.readYaml('config.yaml') - config = dict() - for item in data['config']: - config.update(item) - - timeZone = str(config['timeZone']) - - # 获取程序启动时间 - startTime = datetime.datetime.now(tz=pytz.timezone(timeZone)).strftime("%Y-%m-%d %H:%M:%S") - - # 获得当前的日期-小时向下取整 - tailTime = datetime.datetime.now(tz=pytz.timezone(timeZone)).strftime("%Y-%m-%d %H") - - # 获得待测窗口时间 - headTime = (datetime.datetime.now(tz=pytz.timezone(timeZone)) - datetime.timedelta( - hours=config['hours_N'])).strftime("%Y-%m-%d %H") - - # 获得时间窗口 - headTime = datetime.datetime.strptime(str(headTime) + ':00:00', '%Y-%m-%d %H:%M:%S') - tailTime = datetime.datetime.strptime(str(tailTime) + ':00:00', '%Y-%m-%d %H:%M:%S') - config['headTime'] = headTime - config['tailTime'] = tailTime - - - # 设置测试时间 - if config['ifTest'] == 1: - tailTime = datetime.datetime.strptime(str(config['testEnvironmentTailTime']), '%Y-%m-%d %H:%M:%S').strftime( - "%Y-%m-%d %H") - headTime = config['headTime'] = datetime.datetime.strptime(tailTime, '%Y-%m-%d %H') - datetime.timedelta( - hours=config['hours_N']) - tailTime = datetime.datetime.strptime(str(tailTime) + ':00:00', "%Y-%m-%d %H:%M:%S") - config['tailTime'] = tailTime - - logger.info("DataQuerying from {} to {} ".format(headTime, tailTime)) - - # 配置为非模型训练模式但无model文件时报错退出 - if config['initModelTrain'] == 0: - if not toolFunction.fileExists(config['ModelPath']): - logger.error('Model file "{}" is not found, check if configuration is correct!'.format(config['ModelPath'])) - sys.exit() - # 配置为模型训练模式时进行模型训练 - if config['initModelTrain'] == 1: - logger.info("Start model generation...") - config['headTime'] = config['training_start_time'] - config['tailTime'] = config['training_end_time'] - ModelTraining(config).training() - logger.info("Model-training completed!") - sys.exit() - - # 监控 - if config['monitor_switch'] == 'on': - with open(config['monitor_file_path'], "w") as file: - # 写入一个空字符串作为文件内容 - file.write("") - kb_monitor_metrics = Monitor(config).calculate_kb_metric() - for item in kb_monitor_metrics.items(): - with open(config['monitor_file_path'], "a") as file: - file.write('vpn_thwarting_psiphon3_serverip_' + item[0] + ' ' + str(item[1]) + '\n') - logger.info("[Monitor] - vpn_thwarting_psiphon3_serverip_{} {}".format(item[0], str(item[1]))) - ck_monitor_metrics = Monitor(config).calculate_ck_metric() - for item in ck_monitor_metrics.items(): - with open(config['monitor_file_path'], "a") as file: - file.write('vpn_thwarting_psiphon3_server_app_' + item[0] + ' ' + str(item[1]) + '\n') - logger.info("[Monitor] - vpn_thwarting_psiphon3_server_app_{} {}".format(item[0], str(item[1]))) - - # 配置为非训练模式且已有model文件时进行样本评估 - connectTest = toolFunction.connectTest(config) - detection = ModelDetection(config) - flag = connectTest.dataTest() - if flag == 0: - logger.error("No data in time range above! ") - if flag == 1: - logger.info("Start detection processing ...") - detection.detection() - - if config['if_update_kb']: - detection.upload_to_kb() - logger.info("Complete update knowledgeBase!") - - # 获取程序结束时间 - endTime = datetime.datetime.now(tz=pytz.timezone(timeZone)).strftime("%Y-%m-%d %H:%M:%S") diff --git a/detection/vpnservices/hotspotvpn_serverip.py b/detection/vpnservices/hotspotvpn_serverip.py index 39aa875..e75cff3 100644 --- a/detection/vpnservices/hotspotvpn_serverip.py +++ b/detection/vpnservices/hotspotvpn_serverip.py @@ -38,7 +38,7 @@ class HotspotvpnServerip(VpnDetector): self.logger.info('Start to query hotspotvpn server ip from clickhouse database') # construct query sql - TIME_FILTER_PATTERN = "(common_recv_time > toDateTime('{$start_time}', '{$time_zone}')) AND(common_recv_time <= toDateTime('{$end_time}', '{$time_zone}'))" + TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', self.config['common']['recv_time_columnname']) time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( self.end_time)).replace("{$time_zone}", self.time_zone) self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) diff --git a/detection/vpnservices/ipvanishvpn_servername.py b/detection/vpnservices/ipvanishvpn_servername.py index 3d82074..f9f7890 100644 --- a/detection/vpnservices/ipvanishvpn_servername.py +++ b/detection/vpnservices/ipvanishvpn_servername.py @@ -39,7 +39,7 @@ class IpvanishvpnServername(VpnDetector): self.logger.info('Start to query ipvanishvpn server name from session record') # construct query sql - TIME_FILTER_PATTERN = "(common_recv_time > toDateTime('{$start_time}', '{$time_zone}')) AND(common_recv_time <= toDateTime('{$end_time}', '{$time_zone}'))" + TIME_FILTER_PATTERN = self.config['common']['time_filter_pattern'].replace('recv_time_columnname', self.config['common']['recv_time_columnname']) time_filter = TIME_FILTER_PATTERN.replace("{$start_time}", str(self.start_time)).replace("{$end_time}", str( self.end_time)).replace("{$time_zone}", self.time_zone) self.sql = self.sql.replace("{$db_name}", self.dbname).replace("{$table_name}", self.table_name) diff --git a/logs/run.log b/logs/run.log deleted file mode 100644 index 16e991f..0000000 --- a/logs/run.log +++ /dev/null @@ -1,15 +0,0 @@ -2024-01-12 18:19:10 - hotspotvpn_serverip.py - INFO - Start to query hotspotvpn server ip from clickhouse database -2024-01-12 18:19:10 - connection.py - WARNING - Error on socket shutdown: [Errno 57] Socket is not connected -2024-01-12 18:19:26 - hotspotvpn_serverip.py - INFO - Start to query hotspotvpn server ip from clickhouse database -2024-01-12 18:19:26 - hotspotvpn_serverip.py - INFO - No hotspotvpn server ip found from clickhouse database -2024-01-12 18:19:32 - hotspotvpn_serverip.py - INFO - Start to query hotspotvpn server ip from clickhouse database -2024-01-12 18:19:32 - hotspotvpn_serverip.py - INFO - No hotspotvpn server ip found from clickhouse database -2024-01-12 18:20:13 - vpn_detector.py - INFO - Start to update data to knowledgebase -2024-01-12 18:20:13 - vpn_detector.py - INFO - [Updating knowledgebase]- ip num:1772 -2024-01-12 18:20:14 - KnowledgeBaseTool.py - INFO - {'code': 200, 'data': {'knowledgeId': 15}, 'time': '2024-01-12 10:20:14', 'message': 'success'} -2024-01-12 18:20:14 - KnowledgeBaseTool.py - INFO - Import succeed. Response code 200. -2024-01-12 18:20:28 - ipvanishvpn_servername.py - INFO - Start to query ipvanishvpn server name from session record -2024-01-12 18:20:28 - ipvanishvpn_servername.py - INFO - Sql for ipvanishvpn_servername: SELECT DISTINCT dns_qname FROM tsg_galaxy_p19.session_record WHERE (common_recv_time > toDateTime('2024-01-12 17:00:00', 'Asia/Shanghai')) AND(common_recv_time <= toDateTime('2024-01-12 18:00:00', 'Asia/Shanghai')) AND dns_qname LIKE '%.vpn.ipvanish.com' -2024-01-12 18:20:28 - ipvanishvpn_servername.py - INFO - No ipvanishvpn server ip found from clickhouse database -2024-01-12 18:39:06 - hotspotvpn_serverip.py - INFO - Start to query hotspotvpn server ip from clickhouse database -2024-01-12 18:39:06 - hotspotvpn_serverip.py - INFO - No hotspotvpn server ip found from clickhouse database |