diff options
| author | 尹姜谊 <[email protected]> | 2024-03-18 13:50:45 +0800 |
|---|---|---|
| committer | 尹姜谊 <[email protected]> | 2024-03-18 13:50:45 +0800 |
| commit | 1ae6493db25b9dcdf0be3a302e4afc2f443458f2 (patch) | |
| tree | 61238de1ed263c9731aa4f8912d37d660a1886f6 | |
| parent | c5f7fd8d6085565005dd36c055324b89195ad444 (diff) | |
Add: 增加IP保护
| -rw-r--r-- | config24.01.yaml | 2 | ||||
| -rw-r--r-- | data/turbovpn/turbovpn-ip_2024-03-13_00.csv | 332 | ||||
| -rw-r--r-- | data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv | 2 | ||||
| -rw-r--r-- | detection/awdb/IP_city_single_WGS84.awdb | bin | 0 -> 663073519 bytes | |||
| -rw-r--r-- | detection/awdb/__init__.py | 46 | ||||
| -rw-r--r-- | detection/awdb/compat.py | 39 | ||||
| -rw-r--r-- | detection/awdb/const.py | 6 | ||||
| -rw-r--r-- | detection/awdb/decoder.py | 168 | ||||
| -rw-r--r-- | detection/awdb/errors.py | 2 | ||||
| -rw-r--r-- | detection/awdb/file.py | 45 | ||||
| -rw-r--r-- | detection/awdb/reader.py | 208 | ||||
| -rw-r--r-- | detection/vpn_detector.py | 36 |
12 files changed, 553 insertions, 333 deletions
diff --git a/config24.01.yaml b/config24.01.yaml index ac9c02c..4008c3a 100644 --- a/config24.01.yaml +++ b/config24.01.yaml @@ -8,6 +8,8 @@ common: switch: on max_workers: 100 max_calls_per_sec: 100 + protected_isp_list: ["google", "谷歌", "cloudflare", "microsoft", "alibaba", "amazon", "facebook","微软", "腾讯"] # isp关键词, 英文小写 + protected_ip_list: ['8.8.8.8', '8.8.4.4', '1.1.1.1', '255.255.255.255', '0.0.0.0', '127.*'] monitor: # monitor_file_path: /Users/joy/Downloads/vpn_thwarting_monitor.prom diff --git a/data/turbovpn/turbovpn-ip_2024-03-13_00.csv b/data/turbovpn/turbovpn-ip_2024-03-13_00.csv index 7c66307..2655f3c 100644 --- a/data/turbovpn/turbovpn-ip_2024-03-13_00.csv +++ b/data/turbovpn/turbovpn-ip_2024-03-13_00.csv @@ -1,333 +1,4 @@ addr_format,ip1,ip2,plugin_id,plugin_name,vpn_service_name,method,confidence,is_valid -Single,23.236.105.7,23.236.105.7,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.241.123,154.12.241.123,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.245.115,154.12.245.115,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.236.105.21,23.236.105.21,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.242.50,154.12.242.50,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,209.145.51.238,209.145.51.238,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,38.75.136.25,38.75.136.25,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.152,146.70.82.152,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.12,128.1.186.12,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.122,217.148.140.122,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.13,146.70.61.13,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.23.54,146.70.23.54,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.120,217.148.140.120,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.254,79.110.54.254,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.119,217.148.140.119,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.252,79.110.54.252,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,167.114.169.249,167.114.169.249,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.247,23.251.121.247,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.253,146.70.61.253,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.242,23.251.121.242,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,38.75.137.51,38.75.137.51,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,134.122.36.18,134.122.36.18,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.212,146.70.82.212,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.23.58,146.70.23.58,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,134.122.36.178,134.122.36.178,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.208,45.84.123.208,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.35,98.96.230.35,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.206.182,135.125.206.182,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.20.67,146.70.20.67,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.45,98.96.230.45,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,31.220.101.228,31.220.101.228,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.203.238,135.125.203.238,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.13,128.1.186.13,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.247,128.14.65.247,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.50,98.96.230.50,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.70.35,146.70.70.35,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,143.110.223.151,143.110.223.151,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.217,128.14.65.217,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.134,128.1.186.134,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.121.74,45.84.121.74,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,167.114.169.66,167.114.169.66,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.17,146.70.61.17,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.250,79.110.54.250,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.203.82,135.125.203.82,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.16,23.251.121.16,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,51.222.111.210,51.222.111.210,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.121.75,45.84.121.75,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.245.112,154.12.245.112,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,167.99.82.240,167.99.82.240,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,207.244.227.154,207.244.227.154,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,209.145.52.27,209.145.52.27,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,209.145.48.181,209.145.48.181,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,164.90.213.108,164.90.213.108,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,209.145.49.184,209.145.49.184,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.53.63.128,154.53.63.128,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.245.86,154.12.245.86,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,139.59.182.234,139.59.182.234,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,104.248.248.240,104.248.248.240,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.244.125,154.12.244.125,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,134.122.69.218,134.122.69.218,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,139.59.169.79,139.59.169.79,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,38.75.136.212,38.75.136.212,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.245.109,154.12.245.109,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.249,79.110.54.249,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,165.22.233.199,165.22.233.199,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.98.207.230,98.98.207.230,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.42,98.96.230.42,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.147,146.70.82.147,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.185,128.1.186.185,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,51.222.86.220,51.222.86.220,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.214,146.70.46.214,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,51.222.86.37,51.222.86.37,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.208,128.14.65.208,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.208,146.70.46.208,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.124,217.148.140.124,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.36,98.96.230.36,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.30.57,146.70.30.57,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,134.122.40.165,134.122.40.165,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.130,128.1.186.130,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,169.150.210.66,169.150.210.66,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.240,23.251.121.240,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,143.198.36.28,143.198.36.28,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.241,23.251.121.241,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.201.38,135.125.201.38,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.202,146.70.82.202,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.248,79.110.54.248,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.207.137,135.125.207.137,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.98.206.224,98.98.206.224,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.227,128.14.65.227,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.248,23.251.121.248,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.49,98.96.230.49,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,51.222.15.215,51.222.15.215,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.123,217.148.140.123,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.117,217.148.140.117,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.132,128.1.186.132,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.236.105.18,23.236.105.18,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.205.212,128.1.205.212,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.205.205,128.1.205.205,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.45,98.96.231.45,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.27,98.96.231.27,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.245,146.70.61.245,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.73,128.1.186.73,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.200.79,135.125.200.79,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,38.75.137.65,38.75.137.65,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.23.56,146.70.23.56,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,185.220.68.120,185.220.68.120,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.125,217.148.140.125,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.160,146.70.46.160,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.249,23.251.121.249,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.20.146,146.70.20.146,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.98.206.219,98.98.206.219,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.20.77,146.70.20.77,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.126,217.148.140.126,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.65,128.1.186.65,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.201.133,135.125.201.133,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,165.22.231.93,165.22.231.93,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.253,79.110.54.253,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.23.60,146.70.23.60,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.121.71,45.84.121.71,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.44,98.96.230.44,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.206.65,135.125.206.65,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,193.118.59.104,193.118.59.104,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,169.150.210.71,169.150.210.71,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.231,128.14.65.231,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.75,45.84.123.75,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.251,79.110.54.251,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.72,45.84.123.72,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,134.122.36.77,134.122.36.77,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.246,23.251.121.246,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.205.199,128.1.205.199,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,207.244.232.201,207.244.232.201,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.236.105.5,23.236.105.5,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.74,45.84.123.74,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,193.118.59.88,193.118.59.88,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,185.220.68.55,185.220.68.55,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,169.150.210.45,169.150.210.45,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,38.75.137.60,38.75.137.60,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.245,79.110.54.245,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.254,23.251.121.254,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.215,146.70.46.215,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,134.122.35.105,134.122.35.105,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.38,98.96.230.38,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.61,23.251.121.61,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,51.222.86.214,51.222.86.214,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.57,146.70.61.57,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,207.244.251.58,207.244.251.58,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.243.137,154.12.243.137,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,51.222.111.138,51.222.111.138,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.118,217.148.140.118,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.51,98.96.230.51,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.205.206,128.1.205.206,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.205.196,128.1.205.196,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.53.60.80,154.53.60.80,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.241.121,154.12.241.121,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,38.75.136.191,38.75.136.191,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.242.250,154.12.242.250,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,144.126.134.244,144.126.134.244,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.196,23.251.121.196,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.253,128.14.65.253,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.221,146.70.46.221,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,143.198.46.78,143.198.46.78,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,143.198.44.78,143.198.44.78,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.242,146.70.61.242,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,104.166.161.254,104.166.161.254,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.159,146.70.82.159,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.13,23.251.121.13,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.23.57,146.70.23.57,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.178,128.14.65.178,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.138,128.1.186.138,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,143.198.46.123,143.198.46.123,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,38.75.137.162,38.75.137.162,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.246,79.110.54.246,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.132,146.70.82.132,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.207.226,135.125.207.226,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,193.118.59.84,193.118.59.84,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.207.151,135.125.207.151,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.157,146.70.46.157,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,164.92.189.152,164.92.189.152,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,46.101.17.150,46.101.17.150,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,167.99.193.127,167.99.193.127,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,165.232.72.176,165.232.72.176,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,159.65.115.50,159.65.115.50,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,178.128.167.91,178.128.167.91,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.29,98.96.231.29,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.243.199,154.12.243.199,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.38,98.96.231.38,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.236.105.4,23.236.105.4,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.242.210,154.12.242.210,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.236.105.13,23.236.105.13,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.34,98.96.231.34,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.243.154,154.12.243.154,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.200,146.70.82.200,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,104.166.161.198,104.166.161.198,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.46,98.96.230.46,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.70.34,146.70.70.34,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.121.73,45.84.121.73,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.218,146.70.46.218,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.194,23.251.121.194,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.248,146.70.61.248,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.207,45.84.123.207,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.77,45.84.123.77,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.177,128.14.65.177,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.98.206.220,98.98.206.220,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.158,146.70.82.158,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,79.110.54.247,79.110.54.247,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.40,98.96.230.40,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.244,23.251.121.244,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.20.74,146.70.20.74,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,185.220.68.185,185.220.68.185,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.202.27,135.125.202.27,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.250,128.14.65.250,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.205,146.70.46.205,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.43,98.96.230.43,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,167.99.193.240,167.99.193.240,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,178.128.165.178,178.128.165.178,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,104.248.27.109,104.248.27.109,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,165.232.74.225,165.232.74.225,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,161.35.194.62,161.35.194.62,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,178.128.40.45,178.128.40.45,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.236.105.20,23.236.105.20,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.31,98.96.231.31,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.43,98.96.231.43,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.245.90,154.12.245.90,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.42,98.96.231.42,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.137,128.1.186.137,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,169.150.210.242,169.150.210.242,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.237,146.70.46.237,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.41,146.70.61.41,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,185.220.68.57,185.220.68.57,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.229,128.14.65.229,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,165.22.231.12,165.22.231.12,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.73,45.84.123.73,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.200,23.251.121.200,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.203,45.84.123.203,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.32,98.96.230.32,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.8,128.1.186.8,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.205,146.70.82.205,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.204,146.70.46.204,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.233,146.70.46.233,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.98.206.230,98.98.206.230,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.241,128.14.65.241,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.98.207.217,98.98.207.217,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,207.154.216.229,207.154.216.229,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,161.35.201.176,161.35.201.176,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,139.59.179.69,139.59.179.69,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,46.101.17.112,46.101.17.112,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,64.226.92.77,64.226.92.77,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.245.96,154.12.245.96,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,167.99.86.86,167.99.86.86,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,164.92.230.173,164.92.230.173,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.33,98.96.231.33,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,38.75.136.228,38.75.136.228,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,167.172.174.84,167.172.174.84,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,138.68.145.25,138.68.145.25,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.36,98.96.231.36,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.67,128.1.186.67,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,169.150.210.73,169.150.210.73,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,193.118.59.95,193.118.59.95,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.213,146.70.46.213,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.246,128.14.65.246,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.251,23.251.121.251,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,135.125.204.179,135.125.204.179,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,134.122.36.135,134.122.36.135,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.6,128.1.186.6,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,217.148.140.121,217.148.140.121,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.25,146.70.61.25,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.76,45.84.123.76,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,104.166.161.210,104.166.161.210,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.48,98.96.230.48,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.252,128.14.65.252,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.15,23.251.121.15,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,169.150.210.72,169.150.210.72,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,46.101.17.140,46.101.17.140,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,154.12.245.123,154.12.245.123,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,38.75.136.245,38.75.136.245,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,104.166.161.209,104.166.161.209,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.121.77,45.84.121.77,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,169.150.210.40,169.150.210.40,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.121,128.1.186.121,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.14,23.251.121.14,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.97,98.96.231.97,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,165.232.122.228,165.232.122.228,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.106,98.96.231.106,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,139.59.185.54,139.59.185.54,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.99,98.96.231.99,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,164.90.215.200,164.90.215.200,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,207.244.234.246,207.244.234.246,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,139.59.189.206,139.59.189.206,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.205.200,128.1.205.200,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,64.226.86.237,64.226.86.237,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.24,146.70.61.24,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.23.59,146.70.23.59,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,169.150.210.243,169.150.210.243,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.20.72,146.70.20.72,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.50,146.70.61.50,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.230.39,98.96.230.39,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.135,146.70.82.135,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.131,128.1.186.131,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.225,128.14.65.225,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,104.166.161.195,104.166.161.195,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.121.72,45.84.121.72,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.193,23.251.121.193,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,134.122.40.151,134.122.40.151,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.14.65.210,128.14.65.210,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.46,98.96.231.46,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.236.105.12,23.236.105.12,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.70.41,146.70.70.41,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.199,23.251.121.199,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.20.143,146.70.20.143,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.61.5,146.70.61.5,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,45.84.123.204,45.84.123.204,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.134,146.70.82.134,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.139,128.1.186.139,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.98.207.165,98.98.207.165,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.98.207.162,98.98.207.162,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.251.121.18,23.251.121.18,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,165.22.231.31,165.22.231.31,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.196,146.70.46.196,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,98.96.231.35,98.96.231.35,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,23.236.105.8,23.236.105.8,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,46.101.17.120,46.101.17.120,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,164.92.203.196,164.92.203.196,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,138.68.158.199,138.68.158.199,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,207.154.210.154,207.154.210.154,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,128.1.186.186,128.1.186.186,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.82.151,146.70.82.151,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,146.70.46.217,146.70.46.217,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,64.226.94.247,64.226.94.247,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,167.99.82.112,167.99.82.112,7,turbovpn,turbovpn,passive_ml,confirmed,1 Single,157.245.218.82,157.245.218.82,7,turbovpn,turbovpn,passive_ml,confirmed,1 Single,162.243.5.152,162.243.5.152,7,turbovpn,turbovpn,passive_ml,confirmed,1 Single,104.248.15.166,104.248.15.166,7,turbovpn,turbovpn,passive_ml,confirmed,1 @@ -343,7 +14,4 @@ Single,167.71.102.90,167.71.102.90,7,turbovpn,turbovpn,passive_ml,confirmed,1 Single,143.198.168.208,143.198.168.208,7,turbovpn,turbovpn,passive_ml,confirmed,1 Single,198.199.114.225,198.199.114.225,7,turbovpn,turbovpn,passive_ml,confirmed,1 Single,162.243.5.116,162.243.5.116,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,96.126.100.78,96.126.100.78,7,turbovpn,turbovpn,passive_ml,confirmed,1 Single,162.243.1.80,162.243.1.80,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,167.71.248.241,167.71.248.241,7,turbovpn,turbovpn,passive_ml,confirmed,1 -Single,185.220.68.233,185.220.68.233,7,turbovpn,turbovpn,passive_ml,confirmed,1 diff --git a/data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv b/data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv index 33ac938..b7e16c0 100644 --- a/data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv +++ b/data/windscribevpn/windscribevpn-domain_2024-03-18_09.csv @@ -56,7 +56,6 @@ gr-005.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 hk-001.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 hk-006.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 hr-002.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 -https,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 hu-001.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 id-007.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 id-009.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 @@ -252,6 +251,7 @@ hk-006.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 hk-007.totallyacdn.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 hk-009.totallyacdn.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 hr-002.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 +https,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 hu-001.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 id-002.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 id-007.whiskergalaxy.com,6,windscribevpn,windscribevpn,passive_ml,confirmed,1 diff --git a/detection/awdb/IP_city_single_WGS84.awdb b/detection/awdb/IP_city_single_WGS84.awdb Binary files differnew file mode 100644 index 0000000..26476d9 --- /dev/null +++ b/detection/awdb/IP_city_single_WGS84.awdb diff --git a/detection/awdb/__init__.py b/detection/awdb/__init__.py new file mode 100644 index 0000000..90ea93f --- /dev/null +++ b/detection/awdb/__init__.py @@ -0,0 +1,46 @@ +''' +Description: +Author: chenxu +Date: 2022-09-20 13:16:05 +LastEditTime: 2022-09-20 13:26:49 +LastEditors: yinjiangyi +''' +import os + +import sys +sys.path.append('.') +import awdb.reader + +try: + import awdb.extension +except ImportError: + awdb.extension = None + +from awdb.const import (MODE_AUTO, MODE_MMAP, MODE_MMAP_EXT, MODE_FILE, + MODE_MEMORY, MODE_FD) +from awdb.decoder import InvalidDatabaseError + + +def open_database(database, mode=MODE_AUTO): + has_extension = awdb.extension and hasattr(awdb.extension, + 'Reader') + if (mode == MODE_AUTO and has_extension) or mode == MODE_MMAP_EXT: + if not has_extension: + raise ValueError( + "MODE_MMAP_EXT requires the awdb.extension module to be available" + ) + return awdb.extension.Reader(database) + if mode in (MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY, MODE_FD): + return awdb.reader.Reader(database, mode) + raise ValueError('Unsupported open mode: {0}'.format(mode)) + + +def Reader(database): + return open_database(database) + + +__title__ = 'awdb' +__version__ = '1.5.2' +__author__ = '' +__license__ = 'Apache License, Version 2.0' +__copyright__ = 'Copyright 2013-2020 AW, Inc.' diff --git a/detection/awdb/compat.py b/detection/awdb/compat.py new file mode 100644 index 0000000..9952ac4 --- /dev/null +++ b/detection/awdb/compat.py @@ -0,0 +1,39 @@ +import sys + +import ipaddress + + +if sys.version_info[0] == 2: + + def compat_ip_address(address): + if isinstance(address, bytes): + address = address.decode() + return ipaddress.ip_address(address) + + int_from_byte = ord + + FileNotFoundError = IOError + + def int_from_bytes(b): + if b: + return int(b.encode("hex"), 16) + return 0 + + byte_from_int = chr + + string_type = basestring + +else: + + def compat_ip_address(address): + return ipaddress.ip_address(address) + + int_from_byte = lambda x: x + + FileNotFoundError = FileNotFoundError + + int_from_bytes = lambda x: int.from_bytes(x, 'big') + + byte_from_int = lambda x: bytes([x]) + + string_type = str diff --git a/detection/awdb/const.py b/detection/awdb/const.py new file mode 100644 index 0000000..8618d58 --- /dev/null +++ b/detection/awdb/const.py @@ -0,0 +1,6 @@ +MODE_AUTO = 0 +MODE_MMAP_EXT = 1 +MODE_MMAP = 2 +MODE_FILE = 4 +MODE_MEMORY = 8 +MODE_FD = 16 diff --git a/detection/awdb/decoder.py b/detection/awdb/decoder.py new file mode 100644 index 0000000..1b43fc8 --- /dev/null +++ b/detection/awdb/decoder.py @@ -0,0 +1,168 @@ +from __future__ import unicode_literals + +import struct + +from awdb.compat import byte_from_int, int_from_byte, int_from_bytes +from awdb.errors import InvalidDatabaseError + + +class Decoder(object): + def __init__(self, database_buffer, pointer_base=0, pointer_test=False): + self._pointer_test = pointer_test + self._buffer = database_buffer + self._pointer_base = pointer_base + + def _decode_array(self, size, offset): + array = [] + for _ in range(size): + (value, offset) = self.decode(offset) + array.append(value) + return array, offset + + def _decode_boolean(self, size, offset): + return size != 0, offset + + def _decode_bytes(self, size, offset): + new_offset = offset + size + return self._buffer[offset:new_offset], new_offset + + def _decode_double(self, size, offset): + self._verify_size(size, 8) + new_offset = offset + size + packed_bytes = self._buffer[offset:new_offset] + (value, ) = struct.unpack(b'!d', packed_bytes) + return value, new_offset + + def _decode_float(self, size, offset): + self._verify_size(size, 4) + new_offset = offset + size + packed_bytes = self._buffer[offset:new_offset] + (value, ) = struct.unpack(b'!f', packed_bytes) + return value, new_offset + + def _decode_int32(self, size, offset): + if size == 0: + return 0, offset + new_offset = offset + size + packed_bytes = self._buffer[offset:new_offset] + + if size != 4: + packed_bytes = packed_bytes.rjust(4, b'\x00') + (value, ) = struct.unpack(b'!i', packed_bytes) + return value, new_offset + + def _decode_map(self, size, offset): + container = {} + for _ in range(size): + (key, offset) = self.decode(offset) + (value, offset) = self.decode(offset) + if key == value: + container[key] = bytes(value, 'utf-8') + else: + container[key] = value + # print("###") + # print(container) + # print("###") + return container, offset + + def _decode_pointer(self, size, offset): + pointer_size = (size >> 3) + 1 + + buf = self._buffer[offset:offset + pointer_size] + new_offset = offset + pointer_size + + if pointer_size == 1: + buf = byte_from_int(size & 0x7) + buf + pointer = struct.unpack(b'!H', buf)[0] + self._pointer_base + elif pointer_size == 2: + buf = b'\x00' + byte_from_int(size & 0x7) + buf + pointer = struct.unpack(b'!I', buf)[0] + 2048 + self._pointer_base + elif pointer_size == 3: + buf = byte_from_int(size & 0x7) + buf + pointer = struct.unpack(b'!I', + buf)[0] + 526336 + self._pointer_base + else: + pointer = struct.unpack(b'!I', buf)[0] + self._pointer_base + + if self._pointer_test: + return pointer, new_offset + (value, _) = self.decode(pointer) + return value, new_offset + + def _decode_uint(self, size, offset): + new_offset = offset + size + uint_bytes = self._buffer[offset:new_offset] + return int_from_bytes(uint_bytes), new_offset + + def _decode_utf8_string(self, size, offset): + new_offset = offset + size + return self._buffer[offset:new_offset].decode('utf-8'), new_offset + + _type_decoder = { + 1: _decode_pointer, + 2: _decode_utf8_string, + 3: _decode_double, + 4: _decode_bytes, + 5: _decode_uint, + 6: _decode_uint, + 7: _decode_map, + 8: _decode_int32, + 9: _decode_uint, + 10: _decode_uint, + 11: _decode_array, + 14: _decode_boolean, + 15: _decode_float, + } + + def decode(self, offset): + new_offset = offset + 1 + ctrl_byte = int_from_byte(self._buffer[offset]) + type_num = ctrl_byte >> 5 + if not type_num: + (type_num, new_offset) = self._read_extended(new_offset) + + try: + decoder = self._type_decoder[type_num] + except KeyError: + raise InvalidDatabaseError('Unexpected type number ({type}) ' + 'encountered'.format(type=type_num)) + + (size, new_offset) = self._size_from_ctrl_byte(ctrl_byte, new_offset, + type_num) + return decoder(self, size, new_offset) + + def _read_extended(self, offset): + next_byte = int_from_byte(self._buffer[offset]) + type_num = next_byte + 7 + if type_num < 7: + raise InvalidDatabaseError( + 'Something went horribly wrong in the decoder. An ' + 'extended type resolved to a type number < 8 ' + '({type})'.format(type=type_num)) + return type_num, offset + 1 + + def _verify_size(self, expected, actual): + if expected != actual: + raise InvalidDatabaseError( + 'The AW DB file\'s data section contains bad data ' + '(unknown data type or corrupt data)') + + def _size_from_ctrl_byte(self, ctrl_byte, offset, type_num): + size = ctrl_byte & 0x1f + if type_num == 1 or size < 29: + return size, offset + + if size == 29: + size = 29 + int_from_byte(self._buffer[offset]) + return size, offset + 1 + + if size == 30: + new_offset = offset + 2 + size_bytes = self._buffer[offset:new_offset] + size = 285 + struct.unpack(b'!H', size_bytes)[0] + return size, new_offset + + new_offset = offset + 3 + size_bytes = self._buffer[offset:new_offset] + size = struct.unpack(b'!I', b'\x00' + size_bytes)[0] + 65821 + return size, new_offset diff --git a/detection/awdb/errors.py b/detection/awdb/errors.py new file mode 100644 index 0000000..5687c9e --- /dev/null +++ b/detection/awdb/errors.py @@ -0,0 +1,2 @@ +class InvalidDatabaseError(RuntimeError): + pass diff --git a/detection/awdb/file.py b/detection/awdb/file.py new file mode 100644 index 0000000..6d654f2 --- /dev/null +++ b/detection/awdb/file.py @@ -0,0 +1,45 @@ +import os + +try: + from multiprocessing import Lock +except ImportError: + from threading import Lock + + +class FileBuffer(object): + def __init__(self, database): + self._handle = open(database, 'rb') + self._size = os.fstat(self._handle.fileno()).st_size + if not hasattr(os, 'pread'): + self._lock = Lock() + + def __getitem__(self, key): + if isinstance(key, slice): + return self._read(key.stop - key.start, key.start) + if isinstance(key, int): + return self._read(1, key)[0] + raise TypeError("Invalid argument type.") + + def rfind(self, needle, start): + pos = self._read(self._size - start - 1, start).rfind(needle) + if pos == -1: + return pos + return start + pos + + def size(self): + return self._size + + def close(self): + self._handle.close() + + if hasattr(os, 'pread'): + + def _read(self, buffersize, offset): + return os.pread(self._handle.fileno(), buffersize, offset) + + else: + + def _read(self, buffersize, offset): + with self._lock: + self._handle.seek(offset) + return self._handle.read(buffersize) diff --git a/detection/awdb/reader.py b/detection/awdb/reader.py new file mode 100644 index 0000000..242476b --- /dev/null +++ b/detection/awdb/reader.py @@ -0,0 +1,208 @@ +from __future__ import unicode_literals + +try: + import mmap +except ImportError: + mmap = None + + +import struct +from awdb.compat import compat_ip_address, string_type +from awdb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY, MODE_FD +from awdb.decoder import Decoder +from awdb.errors import InvalidDatabaseError +from awdb.file import FileBuffer + + +class Reader(object): + + _DATA_SECTION_SEPARATOR_SIZE = 16 + _METADATA_START_MARKER = b"\xAB\xCD\xEFipplus360.com" + + _ipv4_start = None + + def __init__(self, database, mode=MODE_AUTO): + if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP: + with open(database, 'rb') as db_file: + self._buffer = mmap.mmap(db_file.fileno(), + 0, + access=mmap.ACCESS_READ) + self._buffer_size = self._buffer.size() + filename = database + elif mode in (MODE_AUTO, MODE_FILE): + self._buffer = FileBuffer(database) + self._buffer_size = self._buffer.size() + filename = database + elif mode == MODE_MEMORY: + with open(database, 'rb') as db_file: + self._buffer = db_file.read() + self._buffer_size = len(self._buffer) + filename = database + elif mode == MODE_FD: + self._buffer = database.read() + self._buffer_size = len(self._buffer) + filename = database.name + else: + raise ValueError( + 'Unsupported open mode ({0}). Only MODE_AUTO, MODE_FILE, ' + 'MODE_MEMORY and MODE_FD are supported by the pure Python ' + 'Reader'.format(mode)) + + metadata_start = self._buffer.rfind( + self._METADATA_START_MARKER, max(0, + self._buffer_size - 128 * 1024)) + + if metadata_start == -1: + self.close() + raise InvalidDatabaseError('Error opening database file ({0}). ' + 'Is this a valid AW DB file?' + ''.format(filename)) + + metadata_start += len(self._METADATA_START_MARKER) + metadata_decoder = Decoder(self._buffer, metadata_start) + (metadata, _) = metadata_decoder.decode(metadata_start) + self._metadata = Metadata(**metadata) + + self._decoder = Decoder( + self._buffer, self._metadata.search_tree_size + + self._DATA_SECTION_SEPARATOR_SIZE) + self.closed = False + + def metadata(self): + return self._metadata + + def get(self, ip_address): + (record, _) = self.get_with_prefix_len(ip_address) + return record + + def get_with_prefix_len(self, ip_address): + if isinstance(ip_address, string_type): + address = compat_ip_address(ip_address) + else: + address = ip_address + + try: + packed_address = bytearray(address.packed) + except AttributeError: + raise TypeError('argument 1 must be a string or ipaddress object') + + if address.version == 6 and self._metadata.ip_version == 4: + raise ValueError( + 'Error looking up {0}. You attempted to look up ' + 'an IPv6 address in an IPv4-only database.'.format(ip_address)) + + (pointer, prefix_len) = self._find_address_in_tree(packed_address) + + if pointer: + return self._resolve_data_pointer(pointer), prefix_len + return None, prefix_len + + def _find_address_in_tree(self, packed): + bit_count = len(packed) * 8 + node = self._start_node(bit_count) + node_count = self._metadata.node_count + + i = 0 + while i < bit_count and node < node_count: + bit = 1 & (packed[i >> 3] >> 7 - (i % 8)) + node = self._read_node(node, bit) + i = i + 1 + + if node == node_count: + return 0, i + if node > node_count: + return node, i + + raise InvalidDatabaseError('Invalid node in search tree') + + def _start_node(self, length): + if self._metadata.ip_version != 6 or length == 128: + return 0 + + if self._ipv4_start: + return self._ipv4_start + + node = 0 + for _ in range(96): + if node >= self._metadata.node_count: + break + node = self._read_node(node, 0) + self._ipv4_start = node + return node + + def _read_node(self, node_number, index): + base_offset = node_number * self._metadata.node_byte_size + + record_size = self._metadata.record_size + if record_size == 24: + offset = base_offset + index * 3 + node_bytes = b'\x00' + self._buffer[offset:offset + 3] + elif record_size == 28: + offset = base_offset + 3 * index + node_bytes = bytearray(self._buffer[offset:offset + 4]) + if index: + node_bytes[0] = 0x0F & node_bytes[0] + else: + middle = (0xF0 & node_bytes.pop()) >> 4 + node_bytes.insert(0, middle) + elif record_size == 32: + offset = base_offset + index * 4 + node_bytes = self._buffer[offset:offset + 4] + else: + raise InvalidDatabaseError( + 'Unknown record size: {0}'.format(record_size)) + return struct.unpack(b'!I', node_bytes)[0] + + def _resolve_data_pointer(self, pointer): + resolved = pointer - self._metadata.node_count + \ + self._metadata.search_tree_size + + if resolved >= self._buffer_size: + raise InvalidDatabaseError( + "The AW DB file's search tree is corrupt") + + (data, _) = self._decoder.decode(resolved) + return data + + def close(self): + if type(self._buffer) not in (str, bytes): + self._buffer.close() + self.closed = True + + def __exit__(self, *args): + self.close() + + def __enter__(self): + if self.closed: + raise ValueError('Attempt to reopen a closed AW DB') + return self + + +class Metadata(object): + def __init__(self, **kwargs): + self.node_count = kwargs['node_count'] + self.record_size = kwargs['record_size'] + self.ip_version = kwargs['ip_version'] + self.database_type = kwargs['database_type'] + self.languages = kwargs['languages'] + self.binary_format_major_version = kwargs[ + 'binary_format_major_version'] + self.binary_format_minor_version = kwargs[ + 'binary_format_minor_version'] + self.build_epoch = kwargs['build_epoch'] + self.description = kwargs['description'] + + @property + def node_byte_size(self): + return self.record_size // 4 + + @property + def search_tree_size(self): + return self.node_count * self.node_byte_size + + def __repr__(self): + args = ', '.join('%s=%r' % x for x in self.__dict__.items()) + return '{module}.{class_name}({data})'.format( + module=self.__module__, + class_name=self.__class__.__name__, + data=args) diff --git a/detection/vpn_detector.py b/detection/vpn_detector.py index 9830ae3..78bea7d 100644 --- a/detection/vpn_detector.py +++ b/detection/vpn_detector.py @@ -20,6 +20,8 @@ from clickhouse_driver import Client from tool.Config import Config from tool.KnowledgeBaseTool import KnowledgeApi import concurrent.futures +from tool.Functions import is_valid_ip +import awdb class ServerGroup: @@ -155,6 +157,36 @@ class VpnDetector: return results + def filtered_by_isp(self, original_ip_list, delete_isp_list): + delete_ip_list = [] + reader = awdb.open_database('detection/awdb/IP_city_single_WGS84.awdb') + for ip in original_ip_list: + try: + (record, prefix_len) = reader.get_with_prefix_len(ip) + isp = str(record.get('owner'), 'utf-8') + for isp_key_words in delete_isp_list: + if isp_key_words in isp.strip('"').strip().lower(): + delete_ip_list.append(ip) + except ValueError: + continue + + return self.filtered_by_ip(original_ip_list, delete_ip_list) + + + def filtered_by_ip(self, original_ip_list, delete_ip_list): + for ip in delete_ip_list: + if is_valid_ip(ip): + if ip in original_ip_list: + original_ip_list.remove(ip) + elif ip.endswith('*'): + to_remove_ip = [i for i in original_ip_list if i.startswith(ip.strip('*'))] + if not len(to_remove_ip) == 0: + [original_ip_list.remove(i) for i in to_remove_ip] + else: + self.logger.error("Wrong format in Protected IP List: {} Support format like 8.8.8.8 or 8.8.*".format(ip)) + return original_ip_list + + # 入口函数定义 if __name__ == '__main__': @@ -234,6 +266,10 @@ if __name__ == '__main__': result_group = detector.find_server() for server_group in result_group: result_list = server_group.server_list + if server_group.object_type == 'ip': + result_list = detector.filtered_by_isp(result_list, detector.config['common']['protected_isp_list']) + result_list = detector.filtered_by_ip(result_list, detector.config['common']['protected_ip_list']) + detector.logger.info('[{}] - Filtered by ISP and IP, {} {} left.'.format(detector.plugin_name, len(result_list), server_group.object_type)) if len(result_list) > 0: detector.save_to_knowledgebase(result_list, server_group.object_type, detector.vpn_service_name, detector.plugin_id, |
