summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
author尹姜谊 <[email protected]>2024-03-18 13:50:45 +0800
committer尹姜谊 <[email protected]>2024-03-18 13:50:45 +0800
commit1ae6493db25b9dcdf0be3a302e4afc2f443458f2 (patch)
tree61238de1ed263c9731aa4f8912d37d660a1886f6
parentc5f7fd8d6085565005dd36c055324b89195ad444 (diff)
Add: 增加IP保护
-rw-r--r--config24.01.yaml2
-rw-r--r--data/turbovpn/turbovpn-ip_2024-03-13_00.csv332
-rw-r--r--data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv2
-rw-r--r--detection/awdb/IP_city_single_WGS84.awdbbin0 -> 663073519 bytes
-rw-r--r--detection/awdb/__init__.py46
-rw-r--r--detection/awdb/compat.py39
-rw-r--r--detection/awdb/const.py6
-rw-r--r--detection/awdb/decoder.py168
-rw-r--r--detection/awdb/errors.py2
-rw-r--r--detection/awdb/file.py45
-rw-r--r--detection/awdb/reader.py208
-rw-r--r--detection/vpn_detector.py36
12 files changed, 553 insertions, 333 deletions
diff --git a/config24.01.yaml b/config24.01.yaml
index ac9c02c..4008c3a 100644
--- a/config24.01.yaml
+++ b/config24.01.yaml
@@ -8,6 +8,8 @@ common:
switch: on
max_workers: 100
max_calls_per_sec: 100
+ protected_isp_list: ["google", "谷歌", "cloudflare", "microsoft", "alibaba", "amazon", "facebook","微软", "腾讯"] # isp关键词, 英文小写
+ protected_ip_list: ['8.8.8.8', '8.8.4.4', '1.1.1.1', '255.255.255.255', '0.0.0.0', '127.*']
monitor:
# monitor_file_path: /Users/joy/Downloads/vpn_thwarting_monitor.prom
diff --git a/data/turbovpn/turbovpn-ip_2024-03-13_00.csv b/data/turbovpn/turbovpn-ip_2024-03-13_00.csv
index 7c66307..2655f3c 100644
--- a/data/turbovpn/turbovpn-ip_2024-03-13_00.csv
+++ b/data/turbovpn/turbovpn-ip_2024-03-13_00.csv
@@ -1,333 +1,4 @@
addr_format,ip1,ip2,plugin_id,plugin_name,vpn_service_name,method,confidence,is_valid
-Single,23.236.105.7,23.236.105.7,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.241.123,154.12.241.123,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.245.115,154.12.245.115,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.236.105.21,23.236.105.21,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.242.50,154.12.242.50,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,209.145.51.238,209.145.51.238,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,38.75.136.25,38.75.136.25,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.152,146.70.82.152,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.12,128.1.186.12,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.122,217.148.140.122,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.13,146.70.61.13,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.23.54,146.70.23.54,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.120,217.148.140.120,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.254,79.110.54.254,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.119,217.148.140.119,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.252,79.110.54.252,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,167.114.169.249,167.114.169.249,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.247,23.251.121.247,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.253,146.70.61.253,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.242,23.251.121.242,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,38.75.137.51,38.75.137.51,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,134.122.36.18,134.122.36.18,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.212,146.70.82.212,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.23.58,146.70.23.58,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,134.122.36.178,134.122.36.178,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.208,45.84.123.208,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.35,98.96.230.35,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.206.182,135.125.206.182,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.20.67,146.70.20.67,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.45,98.96.230.45,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,31.220.101.228,31.220.101.228,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.203.238,135.125.203.238,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.13,128.1.186.13,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.247,128.14.65.247,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.50,98.96.230.50,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.70.35,146.70.70.35,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,143.110.223.151,143.110.223.151,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.217,128.14.65.217,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.134,128.1.186.134,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.121.74,45.84.121.74,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,167.114.169.66,167.114.169.66,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.17,146.70.61.17,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.250,79.110.54.250,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.203.82,135.125.203.82,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.16,23.251.121.16,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,51.222.111.210,51.222.111.210,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.121.75,45.84.121.75,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.245.112,154.12.245.112,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,167.99.82.240,167.99.82.240,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,207.244.227.154,207.244.227.154,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,209.145.52.27,209.145.52.27,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,209.145.48.181,209.145.48.181,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,164.90.213.108,164.90.213.108,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,209.145.49.184,209.145.49.184,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.53.63.128,154.53.63.128,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.245.86,154.12.245.86,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,139.59.182.234,139.59.182.234,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,104.248.248.240,104.248.248.240,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.244.125,154.12.244.125,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,134.122.69.218,134.122.69.218,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,139.59.169.79,139.59.169.79,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,38.75.136.212,38.75.136.212,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.245.109,154.12.245.109,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.249,79.110.54.249,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,165.22.233.199,165.22.233.199,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.98.207.230,98.98.207.230,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.42,98.96.230.42,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.147,146.70.82.147,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.185,128.1.186.185,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,51.222.86.220,51.222.86.220,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.214,146.70.46.214,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,51.222.86.37,51.222.86.37,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.208,128.14.65.208,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.208,146.70.46.208,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.124,217.148.140.124,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.36,98.96.230.36,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.30.57,146.70.30.57,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,134.122.40.165,134.122.40.165,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.130,128.1.186.130,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,169.150.210.66,169.150.210.66,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.240,23.251.121.240,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,143.198.36.28,143.198.36.28,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.241,23.251.121.241,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.201.38,135.125.201.38,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.202,146.70.82.202,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.248,79.110.54.248,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.207.137,135.125.207.137,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.98.206.224,98.98.206.224,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.227,128.14.65.227,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.248,23.251.121.248,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.49,98.96.230.49,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,51.222.15.215,51.222.15.215,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.123,217.148.140.123,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.117,217.148.140.117,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.132,128.1.186.132,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.236.105.18,23.236.105.18,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.205.212,128.1.205.212,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.205.205,128.1.205.205,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.45,98.96.231.45,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.27,98.96.231.27,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.245,146.70.61.245,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.73,128.1.186.73,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.200.79,135.125.200.79,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,38.75.137.65,38.75.137.65,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.23.56,146.70.23.56,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,185.220.68.120,185.220.68.120,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.125,217.148.140.125,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.160,146.70.46.160,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.249,23.251.121.249,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.20.146,146.70.20.146,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.98.206.219,98.98.206.219,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.20.77,146.70.20.77,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.126,217.148.140.126,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.65,128.1.186.65,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.201.133,135.125.201.133,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,165.22.231.93,165.22.231.93,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.253,79.110.54.253,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.23.60,146.70.23.60,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.121.71,45.84.121.71,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.44,98.96.230.44,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.206.65,135.125.206.65,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,193.118.59.104,193.118.59.104,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,169.150.210.71,169.150.210.71,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.231,128.14.65.231,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.75,45.84.123.75,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.251,79.110.54.251,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.72,45.84.123.72,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,134.122.36.77,134.122.36.77,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.246,23.251.121.246,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.205.199,128.1.205.199,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,207.244.232.201,207.244.232.201,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.236.105.5,23.236.105.5,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.74,45.84.123.74,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,193.118.59.88,193.118.59.88,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,185.220.68.55,185.220.68.55,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,169.150.210.45,169.150.210.45,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,38.75.137.60,38.75.137.60,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.245,79.110.54.245,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.254,23.251.121.254,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.215,146.70.46.215,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,134.122.35.105,134.122.35.105,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.38,98.96.230.38,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.61,23.251.121.61,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,51.222.86.214,51.222.86.214,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.57,146.70.61.57,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,207.244.251.58,207.244.251.58,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.243.137,154.12.243.137,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,51.222.111.138,51.222.111.138,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.118,217.148.140.118,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.51,98.96.230.51,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.205.206,128.1.205.206,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.205.196,128.1.205.196,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.53.60.80,154.53.60.80,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.241.121,154.12.241.121,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,38.75.136.191,38.75.136.191,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.242.250,154.12.242.250,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,144.126.134.244,144.126.134.244,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.196,23.251.121.196,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.253,128.14.65.253,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.221,146.70.46.221,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,143.198.46.78,143.198.46.78,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,143.198.44.78,143.198.44.78,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.242,146.70.61.242,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,104.166.161.254,104.166.161.254,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.159,146.70.82.159,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.13,23.251.121.13,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.23.57,146.70.23.57,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.178,128.14.65.178,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.138,128.1.186.138,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,143.198.46.123,143.198.46.123,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,38.75.137.162,38.75.137.162,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.246,79.110.54.246,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.132,146.70.82.132,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.207.226,135.125.207.226,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,193.118.59.84,193.118.59.84,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.207.151,135.125.207.151,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.157,146.70.46.157,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,164.92.189.152,164.92.189.152,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,46.101.17.150,46.101.17.150,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,167.99.193.127,167.99.193.127,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,165.232.72.176,165.232.72.176,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,159.65.115.50,159.65.115.50,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,178.128.167.91,178.128.167.91,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.29,98.96.231.29,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.243.199,154.12.243.199,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.38,98.96.231.38,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.236.105.4,23.236.105.4,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.242.210,154.12.242.210,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.236.105.13,23.236.105.13,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.34,98.96.231.34,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.243.154,154.12.243.154,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.200,146.70.82.200,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,104.166.161.198,104.166.161.198,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.46,98.96.230.46,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.70.34,146.70.70.34,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.121.73,45.84.121.73,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.218,146.70.46.218,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.194,23.251.121.194,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.248,146.70.61.248,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.207,45.84.123.207,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.77,45.84.123.77,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.177,128.14.65.177,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.98.206.220,98.98.206.220,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.158,146.70.82.158,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,79.110.54.247,79.110.54.247,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.40,98.96.230.40,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.244,23.251.121.244,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.20.74,146.70.20.74,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,185.220.68.185,185.220.68.185,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.202.27,135.125.202.27,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.250,128.14.65.250,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.205,146.70.46.205,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.43,98.96.230.43,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,167.99.193.240,167.99.193.240,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,178.128.165.178,178.128.165.178,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,104.248.27.109,104.248.27.109,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,165.232.74.225,165.232.74.225,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,161.35.194.62,161.35.194.62,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,178.128.40.45,178.128.40.45,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.236.105.20,23.236.105.20,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.31,98.96.231.31,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.43,98.96.231.43,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.245.90,154.12.245.90,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.42,98.96.231.42,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.137,128.1.186.137,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,169.150.210.242,169.150.210.242,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.237,146.70.46.237,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.41,146.70.61.41,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,185.220.68.57,185.220.68.57,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.229,128.14.65.229,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,165.22.231.12,165.22.231.12,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.73,45.84.123.73,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.200,23.251.121.200,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.203,45.84.123.203,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.32,98.96.230.32,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.8,128.1.186.8,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.205,146.70.82.205,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.204,146.70.46.204,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.233,146.70.46.233,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.98.206.230,98.98.206.230,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.241,128.14.65.241,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.98.207.217,98.98.207.217,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,207.154.216.229,207.154.216.229,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,161.35.201.176,161.35.201.176,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,139.59.179.69,139.59.179.69,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,46.101.17.112,46.101.17.112,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,64.226.92.77,64.226.92.77,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.245.96,154.12.245.96,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,167.99.86.86,167.99.86.86,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,164.92.230.173,164.92.230.173,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.33,98.96.231.33,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,38.75.136.228,38.75.136.228,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,167.172.174.84,167.172.174.84,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,138.68.145.25,138.68.145.25,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.36,98.96.231.36,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.67,128.1.186.67,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,169.150.210.73,169.150.210.73,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,193.118.59.95,193.118.59.95,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.213,146.70.46.213,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.246,128.14.65.246,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.251,23.251.121.251,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,135.125.204.179,135.125.204.179,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,134.122.36.135,134.122.36.135,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.6,128.1.186.6,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,217.148.140.121,217.148.140.121,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.25,146.70.61.25,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.76,45.84.123.76,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,104.166.161.210,104.166.161.210,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.48,98.96.230.48,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.252,128.14.65.252,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.15,23.251.121.15,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,169.150.210.72,169.150.210.72,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,46.101.17.140,46.101.17.140,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,154.12.245.123,154.12.245.123,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,38.75.136.245,38.75.136.245,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,104.166.161.209,104.166.161.209,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.121.77,45.84.121.77,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,169.150.210.40,169.150.210.40,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.121,128.1.186.121,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.14,23.251.121.14,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.97,98.96.231.97,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,165.232.122.228,165.232.122.228,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.106,98.96.231.106,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,139.59.185.54,139.59.185.54,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.99,98.96.231.99,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,164.90.215.200,164.90.215.200,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,207.244.234.246,207.244.234.246,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,139.59.189.206,139.59.189.206,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.205.200,128.1.205.200,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,64.226.86.237,64.226.86.237,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.24,146.70.61.24,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.23.59,146.70.23.59,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,169.150.210.243,169.150.210.243,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.20.72,146.70.20.72,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.50,146.70.61.50,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.230.39,98.96.230.39,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.135,146.70.82.135,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.131,128.1.186.131,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.225,128.14.65.225,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,104.166.161.195,104.166.161.195,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.121.72,45.84.121.72,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.193,23.251.121.193,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,134.122.40.151,134.122.40.151,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.14.65.210,128.14.65.210,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.46,98.96.231.46,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.236.105.12,23.236.105.12,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.70.41,146.70.70.41,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.199,23.251.121.199,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.20.143,146.70.20.143,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.61.5,146.70.61.5,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,45.84.123.204,45.84.123.204,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.134,146.70.82.134,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.139,128.1.186.139,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.98.207.165,98.98.207.165,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.98.207.162,98.98.207.162,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.251.121.18,23.251.121.18,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,165.22.231.31,165.22.231.31,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.196,146.70.46.196,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,98.96.231.35,98.96.231.35,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,23.236.105.8,23.236.105.8,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,46.101.17.120,46.101.17.120,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,164.92.203.196,164.92.203.196,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,138.68.158.199,138.68.158.199,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,207.154.210.154,207.154.210.154,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,128.1.186.186,128.1.186.186,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.82.151,146.70.82.151,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,146.70.46.217,146.70.46.217,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,64.226.94.247,64.226.94.247,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,167.99.82.112,167.99.82.112,7,turbovpn,turbovpn,passive_ml,confirmed,1
Single,157.245.218.82,157.245.218.82,7,turbovpn,turbovpn,passive_ml,confirmed,1
Single,162.243.5.152,162.243.5.152,7,turbovpn,turbovpn,passive_ml,confirmed,1
Single,104.248.15.166,104.248.15.166,7,turbovpn,turbovpn,passive_ml,confirmed,1
@@ -343,7 +14,4 @@ Single,167.71.102.90,167.71.102.90,7,turbovpn,turbovpn,passive_ml,confirmed,1
Single,143.198.168.208,143.198.168.208,7,turbovpn,turbovpn,passive_ml,confirmed,1
Single,198.199.114.225,198.199.114.225,7,turbovpn,turbovpn,passive_ml,confirmed,1
Single,162.243.5.116,162.243.5.116,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,96.126.100.78,96.126.100.78,7,turbovpn,turbovpn,passive_ml,confirmed,1
Single,162.243.1.80,162.243.1.80,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,167.71.248.241,167.71.248.241,7,turbovpn,turbovpn,passive_ml,confirmed,1
-Single,185.220.68.233,185.220.68.233,7,turbovpn,turbovpn,passive_ml,confirmed,1
diff --git a/data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv b/data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv
index 33ac938..b7e16c0 100644
--- a/data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv
+++ b/data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv
@@ -56,7 +56,6 @@ gr-005.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
hk-001.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
hk-006.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
hr-002.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
-https,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
hu-001.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
id-007.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
id-009.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
@@ -252,6 +251,7 @@ hk-006.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
hk-007.totallyacdn.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
hk-009.totallyacdn.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
hr-002.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
+https,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
hu-001.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
id-002.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
id-007.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1
diff --git a/detection/awdb/IP_city_single_WGS84.awdb b/detection/awdb/IP_city_single_WGS84.awdb
new file mode 100644
index 0000000..26476d9
--- /dev/null
+++ b/detection/awdb/IP_city_single_WGS84.awdb
Binary files differ
diff --git a/detection/awdb/__init__.py b/detection/awdb/__init__.py
new file mode 100644
index 0000000..90ea93f
--- /dev/null
+++ b/detection/awdb/__init__.py
@@ -0,0 +1,46 @@
+'''
+Description:
+Author: chenxu
+Date: 2022-09-20 13:16:05
+LastEditTime: 2022-09-20 13:26:49
+LastEditors: yinjiangyi
+'''
+import os
+
+import sys
+sys.path.append('.')
+import awdb.reader
+
+try:
+ import awdb.extension
+except ImportError:
+ awdb.extension = None
+
+from awdb.const import (MODE_AUTO, MODE_MMAP, MODE_MMAP_EXT, MODE_FILE,
+ MODE_MEMORY, MODE_FD)
+from awdb.decoder import InvalidDatabaseError
+
+
+def open_database(database, mode=MODE_AUTO):
+ has_extension = awdb.extension and hasattr(awdb.extension,
+ 'Reader')
+ if (mode == MODE_AUTO and has_extension) or mode == MODE_MMAP_EXT:
+ if not has_extension:
+ raise ValueError(
+ "MODE_MMAP_EXT requires the awdb.extension module to be available"
+ )
+ return awdb.extension.Reader(database)
+ if mode in (MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY, MODE_FD):
+ return awdb.reader.Reader(database, mode)
+ raise ValueError('Unsupported open mode: {0}'.format(mode))
+
+
+def Reader(database):
+ return open_database(database)
+
+
+__title__ = 'awdb'
+__version__ = '1.5.2'
+__author__ = ''
+__license__ = 'Apache License, Version 2.0'
+__copyright__ = 'Copyright 2013-2020 AW, Inc.'
diff --git a/detection/awdb/compat.py b/detection/awdb/compat.py
new file mode 100644
index 0000000..9952ac4
--- /dev/null
+++ b/detection/awdb/compat.py
@@ -0,0 +1,39 @@
+import sys
+
+import ipaddress
+
+
+if sys.version_info[0] == 2:
+
+ def compat_ip_address(address):
+ if isinstance(address, bytes):
+ address = address.decode()
+ return ipaddress.ip_address(address)
+
+ int_from_byte = ord
+
+ FileNotFoundError = IOError
+
+ def int_from_bytes(b):
+ if b:
+ return int(b.encode("hex"), 16)
+ return 0
+
+ byte_from_int = chr
+
+ string_type = basestring
+
+else:
+
+ def compat_ip_address(address):
+ return ipaddress.ip_address(address)
+
+ int_from_byte = lambda x: x
+
+ FileNotFoundError = FileNotFoundError
+
+ int_from_bytes = lambda x: int.from_bytes(x, 'big')
+
+ byte_from_int = lambda x: bytes([x])
+
+ string_type = str
diff --git a/detection/awdb/const.py b/detection/awdb/const.py
new file mode 100644
index 0000000..8618d58
--- /dev/null
+++ b/detection/awdb/const.py
@@ -0,0 +1,6 @@
+MODE_AUTO = 0
+MODE_MMAP_EXT = 1
+MODE_MMAP = 2
+MODE_FILE = 4
+MODE_MEMORY = 8
+MODE_FD = 16
diff --git a/detection/awdb/decoder.py b/detection/awdb/decoder.py
new file mode 100644
index 0000000..1b43fc8
--- /dev/null
+++ b/detection/awdb/decoder.py
@@ -0,0 +1,168 @@
+from __future__ import unicode_literals
+
+import struct
+
+from awdb.compat import byte_from_int, int_from_byte, int_from_bytes
+from awdb.errors import InvalidDatabaseError
+
+
+class Decoder(object):
+ def __init__(self, database_buffer, pointer_base=0, pointer_test=False):
+ self._pointer_test = pointer_test
+ self._buffer = database_buffer
+ self._pointer_base = pointer_base
+
+ def _decode_array(self, size, offset):
+ array = []
+ for _ in range(size):
+ (value, offset) = self.decode(offset)
+ array.append(value)
+ return array, offset
+
+ def _decode_boolean(self, size, offset):
+ return size != 0, offset
+
+ def _decode_bytes(self, size, offset):
+ new_offset = offset + size
+ return self._buffer[offset:new_offset], new_offset
+
+ def _decode_double(self, size, offset):
+ self._verify_size(size, 8)
+ new_offset = offset + size
+ packed_bytes = self._buffer[offset:new_offset]
+ (value, ) = struct.unpack(b'!d', packed_bytes)
+ return value, new_offset
+
+ def _decode_float(self, size, offset):
+ self._verify_size(size, 4)
+ new_offset = offset + size
+ packed_bytes = self._buffer[offset:new_offset]
+ (value, ) = struct.unpack(b'!f', packed_bytes)
+ return value, new_offset
+
+ def _decode_int32(self, size, offset):
+ if size == 0:
+ return 0, offset
+ new_offset = offset + size
+ packed_bytes = self._buffer[offset:new_offset]
+
+ if size != 4:
+ packed_bytes = packed_bytes.rjust(4, b'\x00')
+ (value, ) = struct.unpack(b'!i', packed_bytes)
+ return value, new_offset
+
+ def _decode_map(self, size, offset):
+ container = {}
+ for _ in range(size):
+ (key, offset) = self.decode(offset)
+ (value, offset) = self.decode(offset)
+ if key == value:
+ container[key] = bytes(value, 'utf-8')
+ else:
+ container[key] = value
+ # print("###")
+ # print(container)
+ # print("###")
+ return container, offset
+
+ def _decode_pointer(self, size, offset):
+ pointer_size = (size >> 3) + 1
+
+ buf = self._buffer[offset:offset + pointer_size]
+ new_offset = offset + pointer_size
+
+ if pointer_size == 1:
+ buf = byte_from_int(size & 0x7) + buf
+ pointer = struct.unpack(b'!H', buf)[0] + self._pointer_base
+ elif pointer_size == 2:
+ buf = b'\x00' + byte_from_int(size & 0x7) + buf
+ pointer = struct.unpack(b'!I', buf)[0] + 2048 + self._pointer_base
+ elif pointer_size == 3:
+ buf = byte_from_int(size & 0x7) + buf
+ pointer = struct.unpack(b'!I',
+ buf)[0] + 526336 + self._pointer_base
+ else:
+ pointer = struct.unpack(b'!I', buf)[0] + self._pointer_base
+
+ if self._pointer_test:
+ return pointer, new_offset
+ (value, _) = self.decode(pointer)
+ return value, new_offset
+
+ def _decode_uint(self, size, offset):
+ new_offset = offset + size
+ uint_bytes = self._buffer[offset:new_offset]
+ return int_from_bytes(uint_bytes), new_offset
+
+ def _decode_utf8_string(self, size, offset):
+ new_offset = offset + size
+ return self._buffer[offset:new_offset].decode('utf-8'), new_offset
+
+ _type_decoder = {
+ 1: _decode_pointer,
+ 2: _decode_utf8_string,
+ 3: _decode_double,
+ 4: _decode_bytes,
+ 5: _decode_uint,
+ 6: _decode_uint,
+ 7: _decode_map,
+ 8: _decode_int32,
+ 9: _decode_uint,
+ 10: _decode_uint,
+ 11: _decode_array,
+ 14: _decode_boolean,
+ 15: _decode_float,
+ }
+
+ def decode(self, offset):
+ new_offset = offset + 1
+ ctrl_byte = int_from_byte(self._buffer[offset])
+ type_num = ctrl_byte >> 5
+ if not type_num:
+ (type_num, new_offset) = self._read_extended(new_offset)
+
+ try:
+ decoder = self._type_decoder[type_num]
+ except KeyError:
+ raise InvalidDatabaseError('Unexpected type number ({type}) '
+ 'encountered'.format(type=type_num))
+
+ (size, new_offset) = self._size_from_ctrl_byte(ctrl_byte, new_offset,
+ type_num)
+ return decoder(self, size, new_offset)
+
+ def _read_extended(self, offset):
+ next_byte = int_from_byte(self._buffer[offset])
+ type_num = next_byte + 7
+ if type_num < 7:
+ raise InvalidDatabaseError(
+ 'Something went horribly wrong in the decoder. An '
+ 'extended type resolved to a type number < 8 '
+ '({type})'.format(type=type_num))
+ return type_num, offset + 1
+
+ def _verify_size(self, expected, actual):
+ if expected != actual:
+ raise InvalidDatabaseError(
+ 'The AW DB file\'s data section contains bad data '
+ '(unknown data type or corrupt data)')
+
+ def _size_from_ctrl_byte(self, ctrl_byte, offset, type_num):
+ size = ctrl_byte & 0x1f
+ if type_num == 1 or size < 29:
+ return size, offset
+
+ if size == 29:
+ size = 29 + int_from_byte(self._buffer[offset])
+ return size, offset + 1
+
+ if size == 30:
+ new_offset = offset + 2
+ size_bytes = self._buffer[offset:new_offset]
+ size = 285 + struct.unpack(b'!H', size_bytes)[0]
+ return size, new_offset
+
+ new_offset = offset + 3
+ size_bytes = self._buffer[offset:new_offset]
+ size = struct.unpack(b'!I', b'\x00' + size_bytes)[0] + 65821
+ return size, new_offset
diff --git a/detection/awdb/errors.py b/detection/awdb/errors.py
new file mode 100644
index 0000000..5687c9e
--- /dev/null
+++ b/detection/awdb/errors.py
@@ -0,0 +1,2 @@
+class InvalidDatabaseError(RuntimeError):
+ pass
diff --git a/detection/awdb/file.py b/detection/awdb/file.py
new file mode 100644
index 0000000..6d654f2
--- /dev/null
+++ b/detection/awdb/file.py
@@ -0,0 +1,45 @@
+import os
+
+try:
+ from multiprocessing import Lock
+except ImportError:
+ from threading import Lock
+
+
+class FileBuffer(object):
+ def __init__(self, database):
+ self._handle = open(database, 'rb')
+ self._size = os.fstat(self._handle.fileno()).st_size
+ if not hasattr(os, 'pread'):
+ self._lock = Lock()
+
+ def __getitem__(self, key):
+ if isinstance(key, slice):
+ return self._read(key.stop - key.start, key.start)
+ if isinstance(key, int):
+ return self._read(1, key)[0]
+ raise TypeError("Invalid argument type.")
+
+ def rfind(self, needle, start):
+ pos = self._read(self._size - start - 1, start).rfind(needle)
+ if pos == -1:
+ return pos
+ return start + pos
+
+ def size(self):
+ return self._size
+
+ def close(self):
+ self._handle.close()
+
+ if hasattr(os, 'pread'):
+
+ def _read(self, buffersize, offset):
+ return os.pread(self._handle.fileno(), buffersize, offset)
+
+ else:
+
+ def _read(self, buffersize, offset):
+ with self._lock:
+ self._handle.seek(offset)
+ return self._handle.read(buffersize)
diff --git a/detection/awdb/reader.py b/detection/awdb/reader.py
new file mode 100644
index 0000000..242476b
--- /dev/null
+++ b/detection/awdb/reader.py
@@ -0,0 +1,208 @@
+from __future__ import unicode_literals
+
+try:
+ import mmap
+except ImportError:
+ mmap = None
+
+
+import struct
+from awdb.compat import compat_ip_address, string_type
+from awdb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY, MODE_FD
+from awdb.decoder import Decoder
+from awdb.errors import InvalidDatabaseError
+from awdb.file import FileBuffer
+
+
+class Reader(object):
+
+ _DATA_SECTION_SEPARATOR_SIZE = 16
+ _METADATA_START_MARKER = b"\xAB\xCD\xEFipplus360.com"
+
+ _ipv4_start = None
+
+ def __init__(self, database, mode=MODE_AUTO):
+ if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP:
+ with open(database, 'rb') as db_file:
+ self._buffer = mmap.mmap(db_file.fileno(),
+ 0,
+ access=mmap.ACCESS_READ)
+ self._buffer_size = self._buffer.size()
+ filename = database
+ elif mode in (MODE_AUTO, MODE_FILE):
+ self._buffer = FileBuffer(database)
+ self._buffer_size = self._buffer.size()
+ filename = database
+ elif mode == MODE_MEMORY:
+ with open(database, 'rb') as db_file:
+ self._buffer = db_file.read()
+ self._buffer_size = len(self._buffer)
+ filename = database
+ elif mode == MODE_FD:
+ self._buffer = database.read()
+ self._buffer_size = len(self._buffer)
+ filename = database.name
+ else:
+ raise ValueError(
+ 'Unsupported open mode ({0}). Only MODE_AUTO, MODE_FILE, '
+ 'MODE_MEMORY and MODE_FD are supported by the pure Python '
+ 'Reader'.format(mode))
+
+ metadata_start = self._buffer.rfind(
+ self._METADATA_START_MARKER, max(0,
+ self._buffer_size - 128 * 1024))
+
+ if metadata_start == -1:
+ self.close()
+ raise InvalidDatabaseError('Error opening database file ({0}). '
+ 'Is this a valid AW DB file?'
+ ''.format(filename))
+
+ metadata_start += len(self._METADATA_START_MARKER)
+ metadata_decoder = Decoder(self._buffer, metadata_start)
+ (metadata, _) = metadata_decoder.decode(metadata_start)
+ self._metadata = Metadata(**metadata)
+
+ self._decoder = Decoder(
+ self._buffer, self._metadata.search_tree_size +
+ self._DATA_SECTION_SEPARATOR_SIZE)
+ self.closed = False
+
+ def metadata(self):
+ return self._metadata
+
+ def get(self, ip_address):
+ (record, _) = self.get_with_prefix_len(ip_address)
+ return record
+
+ def get_with_prefix_len(self, ip_address):
+ if isinstance(ip_address, string_type):
+ address = compat_ip_address(ip_address)
+ else:
+ address = ip_address
+
+ try:
+ packed_address = bytearray(address.packed)
+ except AttributeError:
+ raise TypeError('argument 1 must be a string or ipaddress object')
+
+ if address.version == 6 and self._metadata.ip_version == 4:
+ raise ValueError(
+ 'Error looking up {0}. You attempted to look up '
+ 'an IPv6 address in an IPv4-only database.'.format(ip_address))
+
+ (pointer, prefix_len) = self._find_address_in_tree(packed_address)
+
+ if pointer:
+ return self._resolve_data_pointer(pointer), prefix_len
+ return None, prefix_len
+
+ def _find_address_in_tree(self, packed):
+ bit_count = len(packed) * 8
+ node = self._start_node(bit_count)
+ node_count = self._metadata.node_count
+
+ i = 0
+ while i < bit_count and node < node_count:
+ bit = 1 & (packed[i >> 3] >> 7 - (i % 8))
+ node = self._read_node(node, bit)
+ i = i + 1
+
+ if node == node_count:
+ return 0, i
+ if node > node_count:
+ return node, i
+
+ raise InvalidDatabaseError('Invalid node in search tree')
+
+ def _start_node(self, length):
+ if self._metadata.ip_version != 6 or length == 128:
+ return 0
+
+ if self._ipv4_start:
+ return self._ipv4_start
+
+ node = 0
+ for _ in range(96):
+ if node >= self._metadata.node_count:
+ break
+ node = self._read_node(node, 0)
+ self._ipv4_start = node
+ return node
+
+ def _read_node(self, node_number, index):
+ base_offset = node_number * self._metadata.node_byte_size
+
+ record_size = self._metadata.record_size
+ if record_size == 24:
+ offset = base_offset + index * 3
+ node_bytes = b'\x00' + self._buffer[offset:offset + 3]
+ elif record_size == 28:
+ offset = base_offset + 3 * index
+ node_bytes = bytearray(self._buffer[offset:offset + 4])
+ if index:
+ node_bytes[0] = 0x0F & node_bytes[0]
+ else:
+ middle = (0xF0 & node_bytes.pop()) >> 4
+ node_bytes.insert(0, middle)
+ elif record_size == 32:
+ offset = base_offset + index * 4
+ node_bytes = self._buffer[offset:offset + 4]
+ else:
+ raise InvalidDatabaseError(
+ 'Unknown record size: {0}'.format(record_size))
+ return struct.unpack(b'!I', node_bytes)[0]
+
+ def _resolve_data_pointer(self, pointer):
+ resolved = pointer - self._metadata.node_count + \
+ self._metadata.search_tree_size
+
+ if resolved >= self._buffer_size:
+ raise InvalidDatabaseError(
+ "The AW DB file's search tree is corrupt")
+
+ (data, _) = self._decoder.decode(resolved)
+ return data
+
+ def close(self):
+ if type(self._buffer) not in (str, bytes):
+ self._buffer.close()
+ self.closed = True
+
+ def __exit__(self, *args):
+ self.close()
+
+ def __enter__(self):
+ if self.closed:
+ raise ValueError('Attempt to reopen a closed AW DB')
+ return self
+
+
+class Metadata(object):
+ def __init__(self, **kwargs):
+ self.node_count = kwargs['node_count']
+ self.record_size = kwargs['record_size']
+ self.ip_version = kwargs['ip_version']
+ self.database_type = kwargs['database_type']
+ self.languages = kwargs['languages']
+ self.binary_format_major_version = kwargs[
+ 'binary_format_major_version']
+ self.binary_format_minor_version = kwargs[
+ 'binary_format_minor_version']
+ self.build_epoch = kwargs['build_epoch']
+ self.description = kwargs['description']
+
+ @property
+ def node_byte_size(self):
+ return self.record_size // 4
+
+ @property
+ def search_tree_size(self):
+ return self.node_count * self.node_byte_size
+
+ def __repr__(self):
+ args = ', '.join('%s=%r' % x for x in self.__dict__.items())
+ return '{module}.{class_name}({data})'.format(
+ module=self.__module__,
+ class_name=self.__class__.__name__,
+ data=args)
diff --git a/detection/vpn_detector.py b/detection/vpn_detector.py
index 9830ae3..78bea7d 100644
--- a/detection/vpn_detector.py
+++ b/detection/vpn_detector.py
@@ -20,6 +20,8 @@ from clickhouse_driver import Client
from tool.Config import Config
from tool.KnowledgeBaseTool import KnowledgeApi
import concurrent.futures
+from tool.Functions import is_valid_ip
+import awdb
class ServerGroup:
@@ -155,6 +157,36 @@ class VpnDetector:
return results
+ def filtered_by_isp(self, original_ip_list, delete_isp_list):
+ delete_ip_list = []
+ reader = awdb.open_database('detection/awdb/IP_city_single_WGS84.awdb')
+ for ip in original_ip_list:
+ try:
+ (record, prefix_len) = reader.get_with_prefix_len(ip)
+ isp = str(record.get('owner'), 'utf-8')
+ for isp_key_words in delete_isp_list:
+ if isp_key_words in isp.strip('"').strip().lower():
+ delete_ip_list.append(ip)
+ except ValueError:
+ continue
+
+ return self.filtered_by_ip(original_ip_list, delete_ip_list)
+
+
+ def filtered_by_ip(self, original_ip_list, delete_ip_list):
+ for ip in delete_ip_list:
+ if is_valid_ip(ip):
+ if ip in original_ip_list:
+ original_ip_list.remove(ip)
+ elif ip.endswith('*'):
+ to_remove_ip = [i for i in original_ip_list if i.startswith(ip.strip('*'))]
+ if not len(to_remove_ip) == 0:
+ [original_ip_list.remove(i) for i in to_remove_ip]
+ else:
+ self.logger.error("Wrong format in Protected IP List: {} Support format like 8.8.8.8 or 8.8.*".format(ip))
+ return original_ip_list
+
+
# 入口函数定义
if __name__ == '__main__':
@@ -234,6 +266,10 @@ if __name__ == '__main__':
result_group = detector.find_server()
for server_group in result_group:
result_list = server_group.server_list
+ if server_group.object_type == 'ip':
+ result_list = detector.filtered_by_isp(result_list, detector.config['common']['protected_isp_list'])
+ result_list = detector.filtered_by_ip(result_list, detector.config['common']['protected_ip_list'])
+ detector.logger.info('[{}] - Filtered by ISP and IP, {} {} left.'.format(detector.plugin_name, len(result_list), server_group.object_type))
if len(result_list) > 0:
detector.save_to_knowledgebase(result_list, server_group.object_type,
detector.vpn_service_name, detector.plugin_id,