From 6ba8db34d295086a294c15f888b6ec0a928e87f4 Mon Sep 17 00:00:00 2001 From: 尹姜谊 Date: Tue, 23 Jan 2024 10:12:59 +0800 Subject: Add: ActiveObtainer --- config.yaml | 129 ------------ config23.10.yaml | 131 ++++++++++++ config24.01.yaml | 131 ++++++++++++ detection/tool/ActiveObtainer.py | 252 +++++++++++++++++++++++ detection/tool/Config.py | 18 +- detection/tool/Functions.py | 79 +++++++ detection/tool/__pycache__/Config.cpython-39.pyc | Bin 1190 -> 1513 bytes version.txt | 0 8 files changed, 608 insertions(+), 132 deletions(-) delete mode 100644 config.yaml create mode 100644 config23.10.yaml create mode 100644 config24.01.yaml create mode 100644 detection/tool/ActiveObtainer.py create mode 100644 version.txt diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 9ac560f..0000000 --- a/config.yaml +++ /dev/null @@ -1,129 +0,0 @@ -common: - output_path: data/ - time_zone: Asia/Shanghai - recv_time_columnname: common_recv_time - time_filter_pattern: (recv_time_columnname> toDateTime('{$start_time}', '{$time_zone}')) AND(recv_time_columnname <= toDateTime('{$end_time}', '{$time_zone}')) - -clickhouse: - host: 192.168.44.30 - port: 9001 - username: default - password: galaxy2019 - db_name: tsg_galaxy_v3 - table_name: session_record - -mariadb: - host: 192.168.44.53 - port: 3306 - user: root - pswd: 111111 - timezone_hour_gap: 8 # actual local timezone - mariadb timezone (hours) - db_name: cn_api - ip_table_name: cn_vpn_learning_ip - domain_table_name: cn_vpn_learning_domain - -knowledgebase: - host: 192.168.44.54:8090 - kb_username: learning_engine - api_pin: 111111 - api_path: /v1/knowledgeBase/items/batch - api_token: a2857bc21b01421b85953fc2c65b4d4c - api_retry_times: 3 - api_timeout: 9999 - db_name: cn_api - ip_library_name: vpn_learning_ip - domain_library_name: vpn_learning_domain - - -hotspotvpn_serverip: - vpn_service_name: hotspotvpn - plugin_id: 1 - plugin_name: hotspotvpn_serverip - object_type: ip - confidence: confirmed - sql: SELECT common_server_ip, any(common_server_asn) AS asn, count(*) AS session_num, groupUniqArray(common_server_domain) as domains, length(domains) as domain_count, countDistinct(common_client_ip) AS cip_num FROM {$db_name}.{$table_name} WHERE {$time_filter} AND (ssl_sni IN ({$domain_list})) GROUP BY common_server_ip having domain_count >= 3 - domains: paypal.com, facebook.com, twitter.com, whatsapp.com, get.adobe.com, cloudfront.net, mozilla.org - - -ipvanishvpn_servername: - vpn_service_name: ipvanishvpn - plugin_id: 2 - plugin_name: ipvanishvpn_servername - object_type: domain - confidence: confirmed - sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.vpn.ipvanish.com' - - -ipvanishvpn_serverip: - vpn_service_name: ipvanishvpn - plugin_id: 3 - plugin_name: ipvanishvpn_serverip - object_type: ip - confidence: confirmed - kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'ipvanishvpn' - - -psiphon3vpn_serverip: - vpn_service_name: psiphon3vpn - plugin_id: 4 - plugin_name: psiphon3vpn_serverip - object_type: ip - confidence: - - -cyberghostvpn_servername: - vpn_service_name: cyberghostvpn - plugin_id: 5 - plugin_name: cyberghostvpn_servername - object_type: domain - confidence: confirmed - sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.nodes.gen4.ninja' - - -cyberghostvpn_serverip: - vpn_service_name: cyberghostvpn - plugin_id: 6 - plugin_name: cyberghostvpn_serverip - object_type: ip - confidence: confirmed - kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'cyberghostvpn' - - -geckovpn_serverip: - vpn_service_name: geckovpn - plugin_id: 7 - plugin_name: geckovpn_serverip - object_type: ip - confidence: confirmed - sql: SELECT DISTINCT common_server_ip FROM {$db_name}.{$table_name} WHERE {$time_filter} AND ssl_cert_issuer like '%CN=SUV;O=SUV999%' - - - -ivacyvpn_servername: - vpn_service_name: ivacyvpn - plugin_id: 8 - plugin_name: ivacyvpn_servername - object_type: domain - confidence: confirmed - sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.pointtoserver.com' - - - -ivacyvpn_serverip: - vpn_service_name: ivacyvpn - plugin_id: 9 - plugin_name: ivacyvpn_serverip - object_type: ip - confidence: confirmed - kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'ivacyvpn' - - -turbovpn_serverip: - vpn_service_name: turbovpn - plugin_id: 10 - plugin_name: turbovpn_serverip - object_type: ip - confidence: confirmed - security_table_name: security_event - security_policy_id: 3847 - sql: SELECT common_server_ip FROM {$db_name}.{$security_table_name} WHERE {$time_filter} AND common_policy_id ={$security_policy_id} AND common_server_port IN (66, 109, 8080, 97, 94, 92) GROUP BY common_server_ip having length(groupUniqArray(common_server_port))>3 \ No newline at end of file diff --git a/config23.10.yaml b/config23.10.yaml new file mode 100644 index 0000000..a35b2fa --- /dev/null +++ b/config23.10.yaml @@ -0,0 +1,131 @@ +common: + output_path: data/ + time_zone: Asia/Shanghai + recv_time_columnname: common_recv_time + time_filter_pattern: (recv_time_columnname> toDateTime('{$start_time}', '{$time_zone}')) AND(recv_time_columnname <= toDateTime('{$end_time}', '{$time_zone}')) + +clickhouse: + host: 192.168.44.30 + port: 9001 + username: default + password: galaxy2019 + db_name: tsg_galaxy_v3 + table_name: session_record + +mariadb: + host: 192.168.44.53 + port: 3306 + user: root + pswd: 111111 + timezone_hour_gap: 8 # actual local timezone - mariadb timezone (hours) + db_name: cn_api + ip_table_name: cn_vpn_learning_ip + domain_table_name: cn_vpn_learning_domain + +knowledgebase: + host: 192.168.44.54:8090 + kb_username: learning_engine + api_pin: 111111 + api_path: /v1/knowledgeBase/items/batch + api_token: a2857bc21b01421b85953fc2c65b4d4c + api_retry_times: 3 + api_timeout: 9999 + db_name: cn_api + ip_library_name: vpn_learning_ip + domain_library_name: vpn_learning_domain + + +### PLUGIN CONFIGS + +hotspotvpn_serverip: + vpn_service_name: hotspotvpn + plugin_id: 1 + plugin_name: hotspotvpn_serverip + object_type: ip + confidence: confirmed + sql: SELECT common_server_ip, any(common_server_asn) AS asn, count(*) AS session_num, groupUniqArray(common_server_domain) as domains, length(domains) as domain_count, countDistinct(common_client_ip) AS cip_num FROM {$db_name}.{$table_name} WHERE {$time_filter} AND (ssl_sni IN ({$domain_list})) GROUP BY common_server_ip having domain_count >= 3 + domains: paypal.com, facebook.com, twitter.com, whatsapp.com, get.adobe.com, cloudfront.net, mozilla.org + + +ipvanishvpn_servername: + vpn_service_name: ipvanishvpn + plugin_id: 2 + plugin_name: ipvanishvpn_servername + object_type: domain + confidence: confirmed + sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.vpn.ipvanish.com' + + +ipvanishvpn_serverip: + vpn_service_name: ipvanishvpn + plugin_id: 3 + plugin_name: ipvanishvpn_serverip + object_type: ip + confidence: confirmed + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'ipvanishvpn' + + +psiphon3vpn_serverip: + vpn_service_name: psiphon3vpn + plugin_id: 4 + plugin_name: psiphon3vpn_serverip + object_type: ip + confidence: + + +cyberghostvpn_servername: + vpn_service_name: cyberghostvpn + plugin_id: 5 + plugin_name: cyberghostvpn_servername + object_type: domain + confidence: confirmed + sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.nodes.gen4.ninja' + + +cyberghostvpn_serverip: + vpn_service_name: cyberghostvpn + plugin_id: 6 + plugin_name: cyberghostvpn_serverip + object_type: ip + confidence: confirmed + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'cyberghostvpn' + + +geckovpn_serverip: + vpn_service_name: geckovpn + plugin_id: 7 + plugin_name: geckovpn_serverip + object_type: ip + confidence: confirmed + sql: SELECT DISTINCT common_server_ip FROM {$db_name}.{$table_name} WHERE {$time_filter} AND ssl_cert_issuer like '%CN=SUV;O=SUV999%' + + + +ivacyvpn_servername: + vpn_service_name: ivacyvpn + plugin_id: 8 + plugin_name: ivacyvpn_servername + object_type: domain + confidence: confirmed + sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.pointtoserver.com' + + + +ivacyvpn_serverip: + vpn_service_name: ivacyvpn + plugin_id: 9 + plugin_name: ivacyvpn_serverip + object_type: ip + confidence: confirmed + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'ivacyvpn' + + +turbovpn_serverip: + vpn_service_name: turbovpn + plugin_id: 10 + plugin_name: turbovpn_serverip + object_type: ip + confidence: confirmed + security_table_name: security_event + security_policy_id: 3847 + sql: SELECT common_server_ip FROM {$db_name}.{$security_table_name} WHERE {$time_filter} AND common_policy_id ={$security_policy_id} AND common_server_port IN (66, 109, 8080, 97, 94, 92, 21, 25) GROUP BY common_server_ip having length(groupUniqArray(common_server_port))>3 \ No newline at end of file diff --git a/config24.01.yaml b/config24.01.yaml new file mode 100644 index 0000000..448dbeb --- /dev/null +++ b/config24.01.yaml @@ -0,0 +1,131 @@ +common: + output_path: data/ + time_zone: Asia/Shanghai + recv_time_columnname: recv_time + time_filter_pattern: (recv_time_columnname> toDateTime('{$start_time}', '{$time_zone}')) AND(recv_time_columnname <= toDateTime('{$end_time}', '{$time_zone}')) + +clickhouse: + host: 192.168.44.30 + port: 9001 + username: default + password: galaxy2019 + db_name: tsg_galaxy_v3 + table_name: session_record + +mariadb: + host: 192.168.44.53 + port: 3306 + user: root + pswd: 111111 + timezone_hour_gap: 8 # actual local timezone - mariadb timezone (hours) + db_name: cn_api + ip_table_name: cn_vpn_learning_ip + domain_table_name: cn_vpn_learning_domain + +knowledgebase: + host: 192.168.44.54:8090 + kb_username: learning_engine + api_pin: 111111 + api_path: /v1/knowledgeBase/items/batch + api_token: a2857bc21b01421b85953fc2c65b4d4c + api_retry_times: 3 + api_timeout: 9999 + db_name: cn_api + ip_library_name: vpn_learning_ip + domain_library_name: vpn_learning_domain + + +### PLUGIN CONFIGS + +hotspotvpn_serverip: + vpn_service_name: hotspotvpn + plugin_id: 1 + plugin_name: hotspotvpn_serverip + object_type: ip + confidence: confirmed + sql: SELECT server_ip, any(server_asn) AS asn, count(*) AS session_num, groupUniqArray(server_domain) as domains, length(domains) as domain_count, countDistinct(client_ip) AS cip_num FROM {$db_name}.{$table_name} WHERE {$time_filter} AND (ssl_sni IN ({$domain_list})) GROUP BY server_ip having domain_count >= 3 + domains: paypal.com, facebook.com, twitter.com, whatsapp.com, get.adobe.com, cloudfront.net, mozilla.org + + +ipvanishvpn_servername: + vpn_service_name: ipvanishvpn + plugin_id: 2 + plugin_name: ipvanishvpn_servername + object_type: domain + confidence: confirmed + sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.vpn.ipvanish.com' + + +ipvanishvpn_serverip: + vpn_service_name: ipvanishvpn + plugin_id: 3 + plugin_name: ipvanishvpn_serverip + object_type: ip + confidence: confirmed + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'ipvanishvpn' + + +psiphon3vpn_serverip: + vpn_service_name: psiphon3vpn + plugin_id: 4 + plugin_name: psiphon3vpn_serverip + object_type: ip + confidence: + + +cyberghostvpn_servername: + vpn_service_name: cyberghostvpn + plugin_id: 5 + plugin_name: cyberghostvpn_servername + object_type: domain + confidence: confirmed + sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.nodes.gen4.ninja' + + +cyberghostvpn_serverip: + vpn_service_name: cyberghostvpn + plugin_id: 6 + plugin_name: cyberghostvpn_serverip + object_type: ip + confidence: confirmed + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'cyberghostvpn' + + +geckovpn_serverip: + vpn_service_name: geckovpn + plugin_id: 7 + plugin_name: geckovpn_serverip + object_type: ip + confidence: confirmed + sql: SELECT DISTINCT server_ip FROM {$db_name}.{$table_name} WHERE {$time_filter} AND ssl_cert_issuer like '%CN=SUV;O=SUV999%' + + + +ivacyvpn_servername: + vpn_service_name: ivacyvpn + plugin_id: 8 + plugin_name: ivacyvpn_servername + object_type: domain + confidence: confirmed + sql: SELECT DISTINCT dns_qname FROM {$db_name}.{$table_name} WHERE {$time_filter} AND dns_qname LIKE '%.pointtoserver.com' + + + +ivacyvpn_serverip: + vpn_service_name: ivacyvpn + plugin_id: 9 + plugin_name: ivacyvpn_serverip + object_type: ip + confidence: confirmed + kb_sql: SELECT distinct domain FROM {$mariadb_dbname}.{$mariadb_domain_tablename} where vpn_service_name = 'ivacyvpn' + + +turbovpn_serverip: + vpn_service_name: turbovpn + plugin_id: 10 + plugin_name: turbovpn_serverip + object_type: ip + confidence: confirmed + security_table_name: security_event + security_policy_id: 3847 + sql: SELECT server_ip FROM {$db_name}.{$security_table_name} WHERE {$time_filter} AND has(security_rule_list, {$security_policy_id}) AND server_port IN (66, 109, 8080, 97, 94, 92, 21, 25) GROUP BY server_ip having length(groupUniqArray(server_port))>3 \ No newline at end of file diff --git a/detection/tool/ActiveObtainer.py b/detection/tool/ActiveObtainer.py new file mode 100644 index 0000000..56c00e0 --- /dev/null +++ b/detection/tool/ActiveObtainer.py @@ -0,0 +1,252 @@ +import argparse +import subprocess +import os +import requests +import json +import csv +import ipaddress + + + +def get_asn_ip_ranges(asn): + """ + Get IP ranges for a given ASN using RIPEstat API. + + Args: + asn (str): The Autonomous System Number (ASN). + + Returns: + list: A list of IP ranges for the given ASN. + """ + url = f"https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS{asn}" + response = requests.get(url) + data = response.json() + + ip_ranges = [] + if "data" in data and "prefixes" in data["data"]: + for item in data["data"]["prefixes"]: + ip_ranges.append(item["prefix"]) + # 创建一个空列表来存储仅 IPv4 地址 + ipv4_list = [] + + for ip in ip_ranges: + try: + # 尝试将字符串解析为 IP 网络地址 + network = ipaddress.ip_network(ip, strict=False) + # 检查是否为 IPv4 地址 + if type(network) is ipaddress.IPv4Network: + ipv4_list.append(ip) + except ValueError: + # 如果地址无效，跳过此地址 + continue + return ipv4_list + + +def run_masscan(ip_list_path, ports, rate='10000', output_path='data/tmp/masscan_output.json'): + # 将端口列表转换为逗号分隔的字符串 + # ports = ','.join(str(port) for port in ports) + + # 设置 Masscan 命令 + masscan_cmd = [ + 'masscan', + '-p', ports, + '--rate', rate, + '-iL', ip_list_path, # 使用文件路径 + '--wait', '5', + '-oJ', output_path # 输出为 JSON 格式 + ] + + # 执行 Masscan 扫描 + print(f"Running masscan with command: {' '.join(masscan_cmd)}") + subprocess.run(masscan_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + with open(output_path, 'r') as file: + data = json.load(file) + port_ip_dict={} + + # 遍历 JSON 数据中的每个条目 + for entry in data: + ip = entry['ip'] + for port_info in entry['ports']: + port = port_info['port'] + # 将 IP 地址添加到对应端口的列表中 + if port in port_ip_dict: + port_ip_dict[port].append(ip) + else: + port_ip_dict[port] = [ip] + + # 打印每个端口及其对应的 IP 地址 + for port, ips in port_ip_dict.items(): + print(f"Port {port}: {ips}") + # 读取 Masscan 输出文件并解析 JSON + #temp_output_path.unlink() + # try: + # subprocess.run(['rm', output_path], check=True) + # print(f"The file {output_path} has been deleted.") + # except subprocess.CalledProcessError: + # print(f"Failed to delete the file {output_path}.") + #subprocess.run(f'rm {temp_output_path}') + return port_ip_dict + + + +def run_zgrab2(port_ip_dict, outfile, tmp_path='/tmp/'): + # 将端口列表转换为逗号分隔的字符串 + # ports = ','.join(str(port) for port in ports) + + # 设置 Zgrab 命令 + for port, ips in port_ip_dict.items(): + with open(tmp_path+f'ips_with_{port}.txt', 'w') as file: + for ip in ips: + file.write(ip + '\n') + strport=str(port) + zgrab2_cmd = [ + 'zgrab2', + 'tls', + '-p', strport, + '-f', tmp_path+f'ips_with_{port}.txt', + '-o', tmp_path+f'zgrab_ips_with_{port}.json', # 使用文件路径 + ] + # 执行 zgrab2 + print(f"Running zgrab2 with command: {' '.join(zgrab2_cmd)}") + result=subprocess.run(zgrab2_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # 打印输出和错误信息 + if result.stdout: + print("ZGrab2 Output:") + print(result.stdout.decode()) + if result.stderr: + print("ZGrab2 Error:") + print(result.stderr.decode()) + json2csv(port_ip_dict,tmp_path+f'zgrab_ips_with_{port}.json','./'+ outfile) + subprocess.run(['rm',tmp_path+f'zgrab_ips_with_{port}.json']) + subprocess.run(['rm',tmp_path+f'ips_with_{port}.txt']) + + + +def json2csv(port_ip_dict,input_json_file,output_csv_file): + def extract_json_objects(file_path): + with open(file_path, 'r', encoding='utf-8') as file: + obj_str = '' # 用于存储当前读取的 JSON 对象字符串 + depth = 0 # 用于跟踪括号深度 + for char in file.read(): + if char == '{': + depth += 1 + if depth > 0: + obj_str += char + if char == '}': + depth -= 1 + if depth == 0: + # 当达到闭合的大括号时，尝试解析 JSON 对象 + try: + yield json.loads(obj_str) + obj_str = '' # 重置字符串以用于下一个对象 + except json.JSONDecodeError as e: + print(f"Error decoding JSON object: {e}") + # 如果出现解码错误，跳过当前对象 + obj_str = '' + + for port, ips in port_ip_dict.items(): + # 准备 CSV 文件 + with open(output_csv_file, 'w', newline='', encoding='utf-8') as csvfile: + fieldnames = ['ip','port', + 'cert_sha1', 'cert_serial_num', 'cert_san', 'cert_cn', + 'cert_issuer', 'cert_issuer_country', 'cert_issuer_organization', + 'cert_subject', 'cert_subject_country', 'cert_subject_organization', + 'cert_start_time', 'cert_expire_time' + ] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for item in extract_json_objects(input_json_file): + # 处理 json_obj + + tls_data = item.get('data', {}).get('tls', {}) + if tls_data.get('status') == 'success': + result = tls_data.get('result', {}) + cert_info = result.get('handshake_log', {}).get('server_certificates', {}).get('certificate', {}).get('parsed', {}) + + # 提取所需字段 + cert_row = { + 'ip': item.get('ip'), + 'port': port, + 'cert_sha1': cert_info.get('fingerprint_sha1'), + 'cert_serial_num': cert_info.get('serial_number'), + 'cert_san': ','.join(cert_info.get('extensions', {}).get('subject_alt_name', {}).get('dns_names', [])), + 'cert_cn': ','.join(cert_info.get('subject', {}).get('common_name', [])), + 'cert_issuer': ','.join(cert_info.get('issuer', {}).get('common_name', [])), + 'cert_issuer_country': ','.join(cert_info.get('issuer', {}).get('country', [])), + 'cert_issuer_organization': ','.join(cert_info.get('issuer', {}).get('organization', [])), + 'cert_subject': cert_info.get('subject_dn'), + 'cert_subject_country': ','.join(cert_info.get('subject', {}).get('country', [])), + 'cert_subject_organization': ','.join(cert_info.get('subject', {}).get('organization', [])), + 'cert_start_time': cert_info.get('validity', {}).get('start'), + 'cert_expire_time': cert_info.get('validity', {}).get('end') + } + + # 写入 CSV + writer.writerow(cert_row) + print(f"{output_csv_file} has been created.") + + +def search_ip_by_ports(ports: object, outfile: object, ip_file: object = None, asns: object = None, rate: object = 10000, port_logic='or') -> object: + """ + Get IP with specific opening ports for a given ASN or a given file + """ + # if ip_file is not None, get iplist from ip_file; if asns is not None, get iplist from asns; else, return error + tmp_path = 'data/tmp/iplist.txt' + ip_ranges = [] + if ip_file is not None: + if not os.path.isfile(ip_file): + print(f"IP list file not found: {ip_file}") + exit(1) + # 读取文件中的 IP 地址 + with open(ip_file, 'r', encoding='utf-8') as file: + ip_ranges = file.read().splitlines() + print("Complete getting ip ranges from file!") + + elif asns is not None: + for asn in asns: + ip_ranges.extend(get_asn_ip_ranges(asn)) + # 检查文件是否存在以及是否以换行符结尾 + print("Complete getting ip ranges from asn!") + + else: + print("Please provide an IP list file or an AS number. Using parameter ip_file or asns") + exit(1) + + # ip_ranges 保存 + with open(tmp_path, 'w', encoding='utf-8') as file: + for ip_range in ip_ranges: + file.write(ip_range + '\n') + + if rate: + port_ip_dict=run_masscan(tmp_path, ports, str(rate)) + else: + port_ip_dict=run_masscan(tmp_path, ports) + + # get values in port_ip_dict and get intersection + selected_ip_list = [] + for port, ips in port_ip_dict.items(): + if len(selected_ip_list) == 0: + selected_ip_list = ips + continue + else: + if port_logic == 'and': + selected_ip_list = set(selected_ip_list).intersection(ips) + elif port_logic == 'or': + selected_ip_list = set(selected_ip_list).union(ips) + selected_ip_list = list(selected_ip_list) + + if outfile is not None: + # 创建文件并保存 + with open(outfile, 'w', encoding='utf-8') as file: + for ip in selected_ip_list: + file.write(ip + '\n') + + return selected_ip_list + + + + + +if __name__ == '__main__': + ip_filtered_by_ports = search_ip_by_ports(ports='554, 22, 53', port_logic='or', asns=['14061'], outfile='data/tmp/filtered_ip.csv', rate='5000') \ No newline at end of file diff --git a/detection/tool/Config.py b/detection/tool/Config.py index 214c3c9..ca23e5f 100644 --- a/detection/tool/Config.py +++ b/detection/tool/Config.py @@ -3,19 +3,31 @@ # @Time : 2024/1/8 12:36 # @author : yinjinagyi # @File : Config.py -# @Function: read config.yaml file and manage all configurations +# @Function: read config23.10.yaml file and manage all configurations import yaml import os import sys + +import LoggingTool + sys.path.append('..') from tool.Functions import get_project_path +logger = LoggingTool.Logger().getLogger() + class Config: def __init__(self): - # read config.yaml file - self.config = self.read_yaml(get_project_path() + '/config.yaml') + # read version information from version.txt, and choose yaml file according to version + with open(get_project_path() + '/version.txt', 'r') as f: + version = f.read().strip() + # 如果文件不存在则报错，日志写入日志文件 + if not os.path.exists(get_project_path() + '/config{}.yaml'.format(version)): + logger.error('config{}.yaml file does not exist'.format(version)) + sys.exit() + + self.config = self.read_yaml(get_project_path() + '/config{}.yaml'.format(version)) # read yaml file diff --git a/detection/tool/Functions.py b/detection/tool/Functions.py index abcee51..e8dc191 100644 --- a/detection/tool/Functions.py +++ b/detection/tool/Functions.py @@ -462,6 +462,85 @@ def get_project_path(): return os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +def get_start_end_ip(ip_range_string): + if '-' in ip_range_string: + start_ip, end_ip = ip_range_string.split('-') + elif '/' in ip_range_string: + network = ipaddress.IPv4Network(ip_range_string, strict=False) + start_ip = str(network.network_address) + end_ip = str(network.broadcast_address) + elif ipaddress.ip_address(ip_range_string): + start_ip = ip_range_string + end_ip = ip_range_string + + if is_valid_public_ip(start_ip) & is_valid_public_ip(end_ip): + return start_ip, end_ip + return + + + +def is_valid_public_ip(ip): + try: + ip_obj = ipaddress.ip_address(ip) + if (ip_obj!=None) & (ip_obj.is_global): + return True + else: + return False + except ValueError: + return False + + +def get_all_ips(start_ip, end_ip): + result = [] + start_ip_obj = ipaddress.ip_address(start_ip) + end_ip_obj = ipaddress.ip_address(end_ip) + + # 遍历起始IP和结束IP之间的所有IP地址 + current_ip = start_ip_obj + while current_ip <= end_ip_obj: + result.append(str(current_ip)) + current_ip += 1 + return result + + +def ipranges_to_ips(ip_ranges): + """ + Discription: 将ipranges转换为ip + :param file_path: ipranges + :return: ip列表 + """ + ip_list = [] + for ip_range in ip_ranges: + result = get_start_end_ip(ip_range) + if len(result) == 2: + start_ip, end_ip = result + ip_list.extend(get_all_ips(start_ip, end_ip)) + ip_list = list(set(ip_list)) + ip_list.sort(key=lambda x: struct.unpack("!L", socket.inet_aton(x))[0]) + + return ip_list + + + +if __name__ == '__main__': + # 读取数据 + ipranges_file_path = '/Users/joy/work/iie/project/cyber_narrator/CN/3-新功能研发/vpn-detection/2.分析脚本/主动爬取/AS14061_IPrange.txt' + output_file_path = '/Users/joy/work/iie/project/cyber_narrator/CN/3-新功能研发/vpn-detection/2.分析脚本/主动爬取/AS14061_IPlists.txt' + + with open(ipranges_file_path, 'r') as file: + lines = file.readlines() + ranges = [i.strip('\n') for i in lines] + ip_ranges = [get_start_end_ip(ip_range) for ip_range in ranges] + ip_list = ipranges_to_ips(ip_ranges) + + # ip_list保存到文件 + with open(output_file_path, 'w') as file: + file.write('\n'.join(ip_list)) + + + + + diff --git a/detection/tool/__pycache__/Config.cpython-39.pyc b/detection/tool/__pycache__/Config.cpython-39.pyc index f1f042c..5a14ac8 100644 Binary files a/detection/tool/__pycache__/Config.cpython-39.pyc and b/detection/tool/__pycache__/Config.cpython-39.pyc differ diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..e69de29 -- cgit v1.2.3