Fix: sys path issue

17 files changed, 33 insertions, 305 deletions

diff --git a/config23.10.yaml b/config23.10.yaml
index 0586509..53ecbee 100644
--- a/config23.10.yaml
+++ b/config23.10.yaml
@@ -141,7 +141,7 @@ geckovpn:
 
 vpnunlimited:
     vpn_service_name: vpnunlimited
-    plugin_id: 11
+    plugin_id: 9
     plugin_name: vpnunlimited
     object_type: ip
     confidence: confirmed
diff --git a/config24.01.yaml b/config24.01.yaml
index 12d2725..b33d9a1 100644
--- a/config24.01.yaml
+++ b/config24.01.yaml
@@ -5,12 +5,13 @@ common:
     time_filter_pattern: (recv_time_columnname> toDateTime('{$start_time}', '{$time_zone}')) AND(recv_time_columnname <= toDateTime('{$end_time}', '{$time_zone}'))
     save_knowledgebase: False
     active_scan:  # max calls/s (rough estimate) = max workers * max_call_per_sec
+        switch: off
         max_workers: 10
         max_calls_per_sec: 10
 
 monitor:
-#    monitor_file_path: /Users/joy/Downloads/vpn_thwarting_monitor.prom
-    monitor_file_path: /opt/vpn-thwarting/vpn_knolwdgebase_monitor.prom
+    monitor_file_path: /Users/joy/Downloads/vpn_thwarting_monitor.prom
+#    monitor_file_path: /home/tsg/olap/galaxy/volumes/node-exporter/prom/vpn_plugin_knowledgebase_monitor.prom
     outdated_days: 100 # outdated after Inactive for days. outdated results will not be monitored as effective results
     timezone_hour_gap: 8
 
diff --git a/data/ivacyvpn/ivacyvpn-domain_2024-02-01_06.csv b/data/ivacyvpn/ivacyvpn-domain_2024-02-01_06.csv
new file mode 100644
index 0000000..d5bc468
--- /dev/null
+++ b/data/ivacyvpn/ivacyvpn-domain_2024-02-01_06.csv
@@ -0,0 +1,12 @@
+domain,plugin_id,plugin_name,vpn_service_name,method,confidence,is_valid
+dc-660.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-666.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-585.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-293.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-555.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-408.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-364.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-84.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-295.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-186.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
+dc-487.pointtoserver.com,3,ivacyvpn,ivacyvpn,passive_ml,confirmed,1
diff --git a/detection/monitor/knowlegebase_monitor.py b/detection/KnowledgeBaseMonitor.py
index a5fbe08..a4e2835 100644
--- a/detection/monitor/knowlegebase_monitor.py
+++ b/detection/KnowledgeBaseMonitor.py
@@ -2,21 +2,14 @@
 # -*- coding: utf-8 -*-
 # @Time    : 2024/1/31 19:41
 # @author    : yinjinagyi
-# @File    : knowlegebase_monitor.py
+# @File    : KnowledgeBaseMonitor.py
 # @Function:
 import argparse
 import datetime
 import numpy as np
 
-import sys
-import os
-
-import pandas as pd
-
-sys.path.append(os.getcwd() + '/detection/tool')
-
-from Config import Config
-from KnowledgeBaseTool import KnowledgeApi, logger
+from tool.Config import Config
+from tool.KnowledgeBaseTool import KnowledgeApi, logger
 
 
 class KnowledgeBaseMonitor:
@@ -192,7 +185,7 @@ if __name__ == '__main__':
             file.write("")
         for item in monitor_result_dict.items():
             with open(monitor_file, "a") as file:
-                file.write('vpn_knowledgebase_monitor_' + item[0] + ' ' + str(item[1]) + '\n')
+                file.write(item[0] + ' ' + str(item[1]) + '\n')
                 logger.info("[Monitor] {}~{} -{} {}".format(start_time, end_time, item[0], str(item[1])))
 
 
diff --git a/detection/tool/ActiveObtainer.py b/detection/tool/ActiveObtainer.py
deleted file mode 100644
index 56c00e0..0000000
--- a/detection/tool/ActiveObtainer.py
+++ /dev/null
@@ -1,252 +0,0 @@
-import argparse
-import subprocess
-import os
-import requests
-import json
-import csv
-import ipaddress
-
-
-
-def get_asn_ip_ranges(asn):
-    """
-    Get IP ranges for a given ASN using RIPEstat API.
-
-    Args:
-    asn (str): The Autonomous System Number (ASN).
-
-    Returns:
-    list: A list of IP ranges for the given ASN.
-    """
-    url = f"https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS{asn}"
-    response = requests.get(url)
-    data = response.json()
-
-    ip_ranges = []
-    if "data" in data and "prefixes" in data["data"]:
-        for item in data["data"]["prefixes"]:
-            ip_ranges.append(item["prefix"])
-    # 创建一个空列表来存储仅 IPv4 地址
-    ipv4_list = []
-
-    for ip in ip_ranges:
-        try:
-            # 尝试将字符串解析为 IP 网络地址
-            network = ipaddress.ip_network(ip, strict=False)
-            # 检查是否为 IPv4 地址
-            if type(network) is ipaddress.IPv4Network:
-                ipv4_list.append(ip)
-        except ValueError:
-            # 如果地址无效，跳过此地址
-            continue
-    return ipv4_list
-
-
-def run_masscan(ip_list_path, ports, rate='10000', output_path='data/tmp/masscan_output.json'):
-    # 将端口列表转换为逗号分隔的字符串
-    # ports = ','.join(str(port) for port in ports)
-
-    # 设置 Masscan 命令
-    masscan_cmd = [
-        'masscan',
-        '-p', ports,
-        '--rate', rate,
-        '-iL', ip_list_path,  # 使用文件路径
-        '--wait', '5',
-        '-oJ', output_path  # 输出为 JSON 格式
-    ]
-
-    # 执行 Masscan 扫描
-    print(f"Running masscan with command: {' '.join(masscan_cmd)}")
-    subprocess.run(masscan_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    with open(output_path, 'r') as file:
-        data = json.load(file)
-    port_ip_dict={}
-
-    # 遍历 JSON 数据中的每个条目
-    for entry in data:
-        ip = entry['ip']
-        for port_info in entry['ports']:
-            port = port_info['port']
-        # 将 IP 地址添加到对应端口的列表中
-            if port in port_ip_dict:
-                port_ip_dict[port].append(ip)
-            else:
-                port_ip_dict[port] = [ip]
-
-    # 打印每个端口及其对应的 IP 地址
-    for port, ips in port_ip_dict.items():
-        print(f"Port {port}: {ips}")
-    # 读取 Masscan 输出文件并解析 JSON
-    #temp_output_path.unlink()
-    # try:
-    #     subprocess.run(['rm', output_path], check=True)
-    #     print(f"The file {output_path} has been deleted.")
-    # except subprocess.CalledProcessError:
-    #     print(f"Failed to delete the file {output_path}.")
-    #subprocess.run(f'rm {temp_output_path}')
-    return port_ip_dict
-
-
-
-def run_zgrab2(port_ip_dict, outfile, tmp_path='/tmp/'):
-    # 将端口列表转换为逗号分隔的字符串
-    # ports = ','.join(str(port) for port in ports)
-
-    # 设置 Zgrab 命令
-    for port, ips in port_ip_dict.items():
-        with open(tmp_path+f'ips_with_{port}.txt', 'w') as file:
-            for ip in ips:
-                file.write(ip + '\n')
-        strport=str(port)
-        zgrab2_cmd = [
-            'zgrab2',
-            'tls',
-            '-p', strport,
-            '-f', tmp_path+f'ips_with_{port}.txt',
-            '-o', tmp_path+f'zgrab_ips_with_{port}.json',  # 使用文件路径
-        ]
-        # 执行 zgrab2
-        print(f"Running zgrab2 with command: {' '.join(zgrab2_cmd)}")
-        result=subprocess.run(zgrab2_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        # 打印输出和错误信息
-        if result.stdout:
-            print("ZGrab2 Output:")
-            print(result.stdout.decode())
-        if result.stderr:
-            print("ZGrab2 Error:")
-            print(result.stderr.decode())
-        json2csv(port_ip_dict,tmp_path+f'zgrab_ips_with_{port}.json','./'+ outfile)
-        subprocess.run(['rm',tmp_path+f'zgrab_ips_with_{port}.json'])
-        subprocess.run(['rm',tmp_path+f'ips_with_{port}.txt'])
-
-
-
-def json2csv(port_ip_dict,input_json_file,output_csv_file):
-    def extract_json_objects(file_path):
-        with open(file_path, 'r', encoding='utf-8') as file:
-            obj_str = ''  # 用于存储当前读取的 JSON 对象字符串
-            depth = 0     # 用于跟踪括号深度
-            for char in file.read():
-                if char == '{':
-                    depth += 1
-                if depth > 0:
-                    obj_str += char
-                if char == '}':
-                    depth -= 1
-                    if depth == 0:
-                        # 当达到闭合的大括号时，尝试解析 JSON 对象
-                        try:
-                            yield json.loads(obj_str)
-                            obj_str = ''  # 重置字符串以用于下一个对象
-                        except json.JSONDecodeError as e:
-                            print(f"Error decoding JSON object: {e}")
-                            # 如果出现解码错误，跳过当前对象
-                            obj_str = ''
-
-    for port, ips in port_ip_dict.items():
-        # 准备 CSV 文件
-        with open(output_csv_file, 'w', newline='', encoding='utf-8') as csvfile:
-            fieldnames = ['ip','port',
-                'cert_sha1', 'cert_serial_num', 'cert_san', 'cert_cn',
-                'cert_issuer', 'cert_issuer_country', 'cert_issuer_organization',
-                'cert_subject', 'cert_subject_country', 'cert_subject_organization',
-                'cert_start_time', 'cert_expire_time'
-            ]
-            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
-            writer.writeheader()
-
-            for item in extract_json_objects(input_json_file):
-                # 处理 json_obj
-
-                tls_data = item.get('data', {}).get('tls', {})
-                if tls_data.get('status') == 'success':
-                    result = tls_data.get('result', {})
-                    cert_info = result.get('handshake_log', {}).get('server_certificates', {}).get('certificate', {}).get('parsed', {})
-
-                    # 提取所需字段
-                    cert_row = {
-                        'ip': item.get('ip'),
-                        'port': port,
-                        'cert_sha1': cert_info.get('fingerprint_sha1'),
-                        'cert_serial_num': cert_info.get('serial_number'),
-                        'cert_san': ','.join(cert_info.get('extensions', {}).get('subject_alt_name', {}).get('dns_names', [])),
-                        'cert_cn': ','.join(cert_info.get('subject', {}).get('common_name', [])),
-                        'cert_issuer': ','.join(cert_info.get('issuer', {}).get('common_name', [])),
-                        'cert_issuer_country': ','.join(cert_info.get('issuer', {}).get('country', [])),
-                        'cert_issuer_organization': ','.join(cert_info.get('issuer', {}).get('organization', [])),
-                        'cert_subject': cert_info.get('subject_dn'),
-                        'cert_subject_country': ','.join(cert_info.get('subject', {}).get('country', [])),
-                        'cert_subject_organization': ','.join(cert_info.get('subject', {}).get('organization', [])),
-                        'cert_start_time': cert_info.get('validity', {}).get('start'),
-                        'cert_expire_time': cert_info.get('validity', {}).get('end')
-                    }
-
-                    # 写入 CSV
-                    writer.writerow(cert_row)
-        print(f"{output_csv_file} has been created.")
-
-
-def search_ip_by_ports(ports: object, outfile: object, ip_file: object = None, asns: object = None, rate: object = 10000, port_logic='or') -> object:
-    """
-    Get IP with specific opening ports for a given ASN or a given file
-    """
-    # if ip_file is not None, get iplist from ip_file; if asns is not None, get iplist from asns; else, return error
-    tmp_path = 'data/tmp/iplist.txt'
-    ip_ranges = []
-    if ip_file is not None:
-        if not os.path.isfile(ip_file):
-            print(f"IP list file not found: {ip_file}")
-            exit(1)
-        # 读取文件中的 IP 地址
-        with open(ip_file, 'r', encoding='utf-8') as file:
-            ip_ranges = file.read().splitlines()
-        print("Complete getting ip ranges from file!")
-
-    elif asns is not None:
-        for asn in asns:
-            ip_ranges.extend(get_asn_ip_ranges(asn))
-        # 检查文件是否存在以及是否以换行符结尾
-        print("Complete getting ip ranges from asn!")
-
-    else:
-        print("Please provide an IP list file or an AS number. Using parameter ip_file or asns")
-        exit(1)
-
-    # ip_ranges 保存
-    with open(tmp_path, 'w', encoding='utf-8') as file:
-        for ip_range in ip_ranges:
-            file.write(ip_range + '\n')
-
-    if rate:
-        port_ip_dict=run_masscan(tmp_path, ports, str(rate))
-    else:
-        port_ip_dict=run_masscan(tmp_path, ports)
-
-    # get values in port_ip_dict and get intersection
-    selected_ip_list = []
-    for port, ips in port_ip_dict.items():
-        if len(selected_ip_list) == 0:
-            selected_ip_list = ips
-            continue
-        else:
-            if port_logic == 'and':
-                selected_ip_list = set(selected_ip_list).intersection(ips)
-            elif port_logic == 'or':
-                selected_ip_list = set(selected_ip_list).union(ips)
-    selected_ip_list = list(selected_ip_list)
-
-    if outfile is not None:
-        # 创建文件并保存
-        with open(outfile, 'w', encoding='utf-8') as file:
-            for ip in selected_ip_list:
-                file.write(ip + '\n')
-
-    return selected_ip_list
-
-
-
-
-
-if __name__ == '__main__':
-    ip_filtered_by_ports = search_ip_by_ports(ports='554, 22, 53', port_logic='or', asns=['14061'], outfile='data/tmp/filtered_ip.csv', rate='5000')
\ No newline at end of file
diff --git a/detection/tool/Config.py b/detection/tool/Config.py
index 9b0d015..b1d412e 100644
--- a/detection/tool/Config.py
+++ b/detection/tool/Config.py
@@ -10,8 +10,8 @@ import os
 import sys
 sys.path.append('..')
 
-from LoggingTool import Logger
-from Functions import get_project_path
+from tool.LoggingTool import Logger
+from tool.Functions import get_project_path
 
 logger = Logger().getLogger()
 
diff --git a/detection/tool/KnowledgeBaseTool.py b/detection/tool/KnowledgeBaseTool.py
index 6fd31f2..0e2f3d1 100644
--- a/detection/tool/KnowledgeBaseTool.py
+++ b/detection/tool/KnowledgeBaseTool.py
@@ -10,9 +10,12 @@ import sys
 import time
 from warnings import simplefilter
 import requests
-import LoggingTool
+import sys
+sys.path.append('..')
+
+from tool.LoggingTool import Logger
 
-logger = LoggingTool.Logger().getLogger()
+logger = Logger().getLogger()
 simplefilter(action='ignore', category=FutureWarning)
 
 
diff --git a/detection/tool/LoggingTool.py b/detection/tool/LoggingTool.py
index b99698e..6650f3a 100644
--- a/detection/tool/LoggingTool.py
+++ b/detection/tool/LoggingTool.py
@@ -5,12 +5,11 @@
 # @File    : LoggingTool.py
 # @Function:
 
-
 import yaml
 import logging.config
 import sys
 sys.path.append('..')
-from Functions import get_project_path
+from tool.Functions import get_project_path
 
 
 class Logger:
diff --git a/detection/tool/ResultEvaluation.py b/detection/tool/ResultEvaluation.py
deleted file mode 100644
index 6ee58df..0000000
--- a/detection/tool/ResultEvaluation.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# @Time    : 2023/3/27 19:34
-# @author    : yinjinagyi
-# @File    : ResultEvaluation.py
-# @Function:
-
-import os
-import pandas as pd
-
-from codev.tool import toolFunction
-
-
-class LabelScraper:
-
-    def __init__(self, start_time, end_time):
-        self.detected_df = self.read_result()
-
-
-    def read_result(self):
-        output_ip_list = []
-        files = toolFunction.filter_files_by_created_time('data/result', '2023-01-01', '2023-03-30')
-        for file in files:
-            output_ip_list.extend(list(pd.read_csv(file, names=['ip', 'pred_y', 'score', 'time'])['ip']))
-
-        result_df = pd.DataFrame(output_ip_list, columns=['ip', 'pred_y', 'score', 'time'])
-
-        return result_df
diff --git a/detection/tool/__pycache__/Config.cpython-39.pyc b/detection/tool/__pycache__/Config.cpython-39.pyc
index 036fa49..ed185db 100644
--- a/detection/tool/__pycache__/Config.cpython-39.pyc
+++ b/detection/tool/__pycache__/Config.cpython-39.pyc
diff --git a/detection/tool/__pycache__/KnowledgeBaseTool.cpython-39.pyc b/detection/tool/__pycache__/KnowledgeBaseTool.cpython-39.pyc
index cf2574d..35fb578 100644
--- a/detection/tool/__pycache__/KnowledgeBaseTool.cpython-39.pyc
+++ b/detection/tool/__pycache__/KnowledgeBaseTool.cpython-39.pyc
diff --git a/detection/tool/__pycache__/LoggingTool.cpython-39.pyc b/detection/tool/__pycache__/LoggingTool.cpython-39.pyc
index 1a54c9d..fd8fa1c 100644
--- a/detection/tool/__pycache__/LoggingTool.cpython-39.pyc
+++ b/detection/tool/__pycache__/LoggingTool.cpython-39.pyc
diff --git a/detection/vpn_detector.py b/detection/vpn_detector.py
index b8f4d3b..8d3c6b4 100644
--- a/detection/vpn_detector.py
+++ b/detection/vpn_detector.py
@@ -46,7 +46,7 @@ class VpnDetector:
         self.config = self.load_config()
         self.output_file_path = self.config['common']['output_path']
         self.logger = Logger().getLogger()
-        self.kb = KnowledgeApi(self.config)
+        self.kb = KnowledgeApi(self.config['knowledgebase'])
         self.start_time = start_time
         self.end_time = end_time
         self.time_zone = self.config['common']['time_zone']
diff --git a/detection/vpnservices/cyberghostvpn.py b/detection/vpnservices/cyberghostvpn.py
index 09ba1af..9cd2eae 100644
--- a/detection/vpnservices/cyberghostvpn.py
+++ b/detection/vpnservices/cyberghostvpn.py
@@ -118,7 +118,7 @@ class CyberghostvpnServerip(VpnDetector):
             servername_list = [i[0] for i in query_result]
 
         # 判断是否能够访问外网，如果能够访问外网，则从外网获取cyberghost_servername_list的域名解析地址
-        if check_internet():
+        if self.config['common']['active_scan']['switch']=='on' and check_internet():
             servername_list = self.find_more_servernames(servername_list)
             if len(servername_list) > 0:
                 resolved_ip_list = self.resolve_dns_for_domain_list(servername_list)
diff --git a/detection/vpnservices/ipvanishvpn.py b/detection/vpnservices/ipvanishvpn.py
index 88f8f1f..5b7905e 100644
--- a/detection/vpnservices/ipvanishvpn.py
+++ b/detection/vpnservices/ipvanishvpn.py
@@ -117,7 +117,7 @@ class IpvanishvpnServerip(VpnDetector):
             servername_list = [i[0] for i in query_result]
 
         # 判断是否能够访问外网，如果能够访问外网，则从外网获取ipvanish_servername_list的域名解析地址
-        if check_internet():
+        if self.config['common']['active_scan']['switch']=='on' and check_internet():
             servername_list = self.find_more_servernames(servername_list)
             if len(servername_list) > 0:
                 resolved_ip_list = self.resolve_dns_for_domain_list(servername_list)
diff --git a/detection/vpnservices/ivacyvpn.py b/detection/vpnservices/ivacyvpn.py
index 805ade1..70ae4a1 100644
--- a/detection/vpnservices/ivacyvpn.py
+++ b/detection/vpnservices/ivacyvpn.py
@@ -100,7 +100,7 @@ class IvacyvpnServerip(VpnDetector):
         server_name_list.extend([f"dc-{str(index)}.pointtoserver.com" for index in range(1000)])
         server_name_list = list(set(server_name_list))
 
-        if check_internet():
+        if self.config['common']['active_scan']['switch']=='on' and check_internet():
             server_ip_list = self.resolve_dns_for_domain_list(server_name_list)
             if server_ip_list:
                 server_ip_list = list(set(server_ip_list))
diff --git a/detection/vpnservices/windscribevpn.py b/detection/vpnservices/windscribevpn.py
index a45c4d9..834a2d0 100644
--- a/detection/vpnservices/windscribevpn.py
+++ b/detection/vpnservices/windscribevpn.py
@@ -120,7 +120,7 @@ class WindscribevpnServerip(VpnDetector):
             servername_list = [i[0] for i in query_result]
 
         # 判断是否能够访问外网，如果能够访问外网，则从外网获取windscribevpn_servername_list的域名解析地址
-        if check_internet():
+        if self.config['common']['active_scan']['switch']=='on' and check_internet():
             servername_list = self.find_more_servernames(servername_list)
             if len(servername_list) > 0:
                 resolved_ip_list = self.resolve_dns_for_domain_list(servername_list)