From 071b1725474b27fe0d5e94d6b239592ab18691ad Mon Sep 17 00:00:00 2001 From: 韩丁康 Date: Sun, 7 Apr 2024 21:51:44 +0800 Subject: 代理主控双向通信功能实现尝试 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/apps/util.py | 9 +++++++++ server/apps/agentcomm.py | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 agent/apps/util.py diff --git a/agent/apps/util.py b/agent/apps/util.py new file mode 100644 index 0000000..986918d --- /dev/null +++ b/agent/apps/util.py @@ -0,0 +1,9 @@ +# 工具函数集合 +import requests +# 代理输出回传 +def agent_echo(proto,server,level,info): + data={"level":level,"info":info} + r=requests.post(proto+"://"+server,json=data) + if r.status_code==200: + print("ok") + diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index cf71e28..3364915 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -43,11 +43,21 @@ def deliver_task(): pass -# 代理输出接收 + @bp.post("/res") @bp.doc("代理输出消息接收接口", hide=True) # TODO:代理输出接收接口实现 -def task_ret(): +def task_ret(json_data): + """ + task_ret 代理输出信息接收 + + Arguments: + json_data -- 请求中的json内容，必须包含"level"和"info"两个keyword + + Returns: + _type_ -- _description_ + """ + print(json_data) return "ok" -- cgit v1.2.3 From c9fa15af845509d1f28fe98e295a0e63a186e15f Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Mon, 8 Apr 2024 14:54:36 +0800 Subject: 代理注册功能开发 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app.py | 56 +++++++++++++++++++++++++++++++++++--------------- agent/apps/util.py | 11 +++++----- agent/requirements.txt | 15 +++++++------- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/agent/app.py b/agent/app.py index d78c893..8216abb 100644 --- a/agent/app.py +++ b/agent/app.py @@ -1,20 +1,22 @@ import argparse import ipaddress +import os.path import platform import socket import geocoder import psutil import requests +import yaml from apiflask import APIFlask -from server.apps.datacheck import bp as datacheckbp from apps.delay import bp as scriptbp # 注册蓝图 app = APIFlask(__name__, template_folder='./static/templates') app.register_blueprint(scriptbp) -app.register_blueprint(datacheckbp) +# 代理配置 +config = {} @app.get('/') @@ -26,8 +28,8 @@ def hello(): # 获取代理的部署环境信息 def nodeinfo(): # IP地址 - ## IPv4地址通过向主控端发包时在主控端获取，从而定位最准确的IPv4地址（公网、内网） - ## IPv6地址获取本地的所有公网地址 + # IPv4地址通过向主控端发包时在主控端获取，从而定位最准确的IPv4地址（公网、内网） + # IPv6地址获取本地的所有公网地址 v6addr = [] addr6_info_list = socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET6) for a in addr6_info_list: @@ -54,21 +56,43 @@ def nodeinfo(): # 注册代理 -def registernode(port=2525, atype="stgj", server="127.0.0.1:8888"): +def registernode(proto="http", port=2525, atype="stgj", server="127.0.0.1:8888"): info = nodeinfo() info["port"] = port info["type"] = atype - requests.post("http://" + server) + r = requests.post(proto + "://" + server, json=info) + if r.status_code != 200: + print("注册失败，请检查参数配置是否正确") + return + return r.json()['id'] if __name__ == '__main__': - # 命令行参数设置 - parser = argparse.ArgumentParser() - parser.add_argument("-p", "--port", type=int, default=2525, help="代理的开放通信端口") - parser.add_argument("-t", "--type", type=str, default="stgj", - help="代理的工作类型 {stgj(渗透攻击) / mbgz(目标感知) / ztgz(状态感知)}") - parser.add_argument("-s", "--server", type=str, default="127.0.0.1:8888", help="主控端访问地址+端口号") - args = parser.parse_args() - registernode(args.port, server=args.server) - - app.run(host="0.0.0.0", debug=True, port=args.port) + # 判断是否已存在配置文件 + if os.path.exists("config.yaml"): + # 存在则读取配置文件信息 + with open("config.yaml", "r") as f: + config = yaml.safe_load(f) + app.run(host="0.0.0.0", port=config["port"], debug=True) + else: + # 不存在则解析命令行参数 + # 命令行参数设置 + parser = argparse.ArgumentParser() + parser.add_argument("-p", "--port", type=int, default=2525, help="代理的开放通信端口") + parser.add_argument("-t", "--type", type=str, default="stgj", + help="代理的工作类型 {stgj(渗透攻击) / mbgz(目标感知) / ztgz(状态感知)}") + parser.add_argument("-s", "--server", type=str, default="127.0.0.1:8888", help="主控端访问地址+端口号") + + # 解析参数 + args = parser.parse_args() + + # 注册代理，并获取主控分配的代理编号 + id = registernode(port=args.port, server=args.server, atype=args.type) + config = {"id": id, "port": args.port, "atype": args.type, "server": args.server} + + # 配置写入yaml文件存储 + with open('config.yaml', 'w') as f: + yaml.dump(config, f) + + # 运行 + app.run(host="0.0.0.0", debug=True, port=config["port"]) diff --git a/agent/apps/util.py b/agent/apps/util.py index 986918d..8cadf4d 100644 --- a/agent/apps/util.py +++ b/agent/apps/util.py @@ -1,9 +1,10 @@ # 工具函数集合 import requests + + # 代理输出回传 -def agent_echo(proto,server,level,info): - data={"level":level,"info":info} - r=requests.post(proto+"://"+server,json=data) - if r.status_code==200: +def agent_echo(proto, server, level, info): + data = {"level": level, "info": info} + r = requests.post(proto + "://" + server, json=data) + if r.status_code == 200: print("ok") - diff --git a/agent/requirements.txt b/agent/requirements.txt index 5494f55..581b4bf 100644 --- a/agent/requirements.txt +++ b/agent/requirements.txt @@ -1,9 +1,10 @@ +requests~=2.31.0 +Flask~=3.0.0 +APIFlask~=2.1.0 click~=8.1.7 -numpy~=1.25.0 +numpy~=1.26.3 six~=1.16.0 -requests~=2.31.0 -pandas~=2.1.1 -APIFlask~=2.0.2 -flask~=3.0.0 -dnspython~=2.4.2 -icmplib~=3.0.4 \ No newline at end of file +pandas~=2.1.4 +psutil~=5.9.0 +geocoder~=1.38.1 +pyyaml~=6.0.1 \ No newline at end of file -- cgit v1.2.3 From bb870fc920277c6abbd5471ac24aa68033abf501 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Mon, 8 Apr 2024 16:10:00 +0800 Subject: 实现代理向主控端注册的功能 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app.py | 21 ++++++++++++++------- agent/requirements.txt | 4 +++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/agent/app.py b/agent/app.py index 8216abb..86e5c87 100644 --- a/agent/app.py +++ b/agent/app.py @@ -4,7 +4,7 @@ import os.path import platform import socket -import geocoder +import ipinfo import psutil import requests import yaml @@ -38,21 +38,26 @@ def nodeinfo(): v6addr.append(a[4][0]) # 经纬度 - g = geocoder.ip('me') - lat, lng = g.lat, g.lng + # 使用ipinfo.io进行定位，调用token + token = "40e1b13cf6b35a" + handler = ipinfo.getHandler(token) + details = handler.getDetails() + # 获取经纬度 + latlng = str(details.loc).split(",") + lat, lng = latlng[0], latlng[1] # CPU核心数 cpu_num = psutil.cpu_count() # 内存信息 - ram_size = psutil.virtual_memory().total + ram_size = int(psutil.virtual_memory().total / (1024 * 1024 * 1024)) ram_per = psutil.virtual_memory().percent # 操作系统 system = platform.system() # return {'v6addr': v6addr, 'lat': lat, 'lng': lng, 'cpu_num': cpu_num, 'ram_size': ram_size, 'ram_per': ram_per, - 'system': system} + 'sys': system} # 注册代理 @@ -60,8 +65,10 @@ def registernode(proto="http", port=2525, atype="stgj", server="127.0.0.1:8888") info = nodeinfo() info["port"] = port info["type"] = atype - r = requests.post(proto + "://" + server, json=info) + print(info) + r = requests.post(proto + "://" + server + "/agent/register", json=info) if r.status_code != 200: + print(r.status_code) print("注册失败，请检查参数配置是否正确") return return r.json()['id'] @@ -73,7 +80,7 @@ if __name__ == '__main__': # 存在则读取配置文件信息 with open("config.yaml", "r") as f: config = yaml.safe_load(f) - app.run(host="0.0.0.0", port=config["port"], debug=True) + app.run(host="0.0.0.0", port=config["port"], debug=True) else: # 不存在则解析命令行参数 # 命令行参数设置 diff --git a/agent/requirements.txt b/agent/requirements.txt index 581b4bf..fbc3eb3 100644 --- a/agent/requirements.txt +++ b/agent/requirements.txt @@ -7,4 +7,6 @@ six~=1.16.0 pandas~=2.1.4 psutil~=5.9.0 geocoder~=1.38.1 -pyyaml~=6.0.1 \ No newline at end of file +pyyaml~=6.0.1 +dnspython~=2.6.1 +icmplib~=3.0.4 \ No newline at end of file -- cgit v1.2.3 From 99d18151d5f2bd980b09d75399be3420c2109daa Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Mon, 8 Apr 2024 16:11:14 +0800 Subject: 更新.gitignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2ead888..000b272 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.out *.pyc ./server/app/__pycache__ +*.yaml -- cgit v1.2.3 From e6b069f5f528c5b85277411f071f462bbf9efda7 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Mon, 8 Apr 2024 16:47:40 +0800 Subject: 主控服务新增配置文件settings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/requirements.txt | 14 +++++++------- server/settings.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 server/settings.py diff --git a/server/requirements.txt b/server/requirements.txt index 5494f55..af9f81b 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,9 +1,9 @@ +requests~=2.31.0 +Flask~=3.0.0 +APIFlask~=2.1.0 click~=8.1.7 -numpy~=1.25.0 +numpy~=1.26.3 six~=1.16.0 -requests~=2.31.0 -pandas~=2.1.1 -APIFlask~=2.0.2 -flask~=3.0.0 -dnspython~=2.4.2 -icmplib~=3.0.4 \ No newline at end of file +pandas~=2.1.4 +faker~=18.9.0 +dnspython~=2.6.1 \ No newline at end of file diff --git a/server/settings.py b/server/settings.py new file mode 100644 index 0000000..5cc1536 --- /dev/null +++ b/server/settings.py @@ -0,0 +1,18 @@ +DEBUG = True +SERVER_PORT = 12526 + +# 数据库配置 +MYSQL_HOST = '127.0.0.1' +MYSQL_PORT = 13306 +MYSQL_PAWD = 'yydns' + +MYSQL_DATADB = "yydnsdata" + +MYSQL_TAB_SYSLOG = "SYSLOG" +MYSQL_TAB_USER = "USER" +MYSQL_TAB_AGENT = "AGENT" +MYSQL_TAB_TARGETDATA = "TARGETDATA" +MYSQL_TAB_TASK = "TASK" +MYSQL_TAB_POLICY = "POLICY" +MYSQL_TAB_TASKPOLICY = "TASK_POLICY" +MYSQL_TASK_LOG = "TASK_LOG" -- cgit v1.2.3 From 36c0bd0f8033e815fd19d7678f3535afe06dc2b7 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Tue, 9 Apr 2024 16:39:47 +0800 Subject: 主控端数据库贯通实现尝试 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app.py | 4 +- server/app.py | 8 +- server/apps/agentcomm.py | 124 +++++++++++++++++++++++------- server/apps/util.py | 191 ++++++++++++++++++++++++++++++++++++++++++++++- server/requirements.txt | 3 +- server/settings.py | 2 +- 6 files changed, 293 insertions(+), 39 deletions(-) diff --git a/agent/app.py b/agent/app.py index 86e5c87..ad85985 100644 --- a/agent/app.py +++ b/agent/app.py @@ -64,7 +64,7 @@ def nodeinfo(): def registernode(proto="http", port=2525, atype="stgj", server="127.0.0.1:8888"): info = nodeinfo() info["port"] = port - info["type"] = atype + info["atype"] = atype print(info) r = requests.post(proto + "://" + server + "/agent/register", json=info) if r.status_code != 200: @@ -86,7 +86,7 @@ if __name__ == '__main__': # 命令行参数设置 parser = argparse.ArgumentParser() parser.add_argument("-p", "--port", type=int, default=2525, help="代理的开放通信端口") - parser.add_argument("-t", "--type", type=str, default="stgj", + parser.add_argument("-t", "--atype", type=str, default="stgj", help="代理的工作类型 {stgj(渗透攻击) / mbgz(目标感知) / ztgz(状态感知)}") parser.add_argument("-s", "--server", type=str, default="127.0.0.1:8888", help="主控端访问地址+端口号") diff --git a/server/app.py b/server/app.py index a53a829..4f02203 100644 --- a/server/app.py +++ b/server/app.py @@ -1,7 +1,6 @@ -import argparse - from apiflask import APIFlask +import settings from apps.agentcomm import bp as agentbp from apps.sysinfo import bp as sysbp from apps.sysmange import bp as mangbp @@ -29,7 +28,4 @@ def hello(): if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-p", "--port", type=int, default=2525, help="主控服务的开放通信端口") - args = parser.parse_args() - app.run(host="0.0.0.0", debug=True, port=args.port) + app.run(host="0.0.0.0", debug=settings.DEBUG, port=settings.SERVER_PORT) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 3364915..d053c2b 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -1,11 +1,13 @@ # 代理通信与注册接口 +import datetime import random from apiflask import APIBlueprint, Schema from apiflask.fields import String, Integer, List, Nested, Boolean, DateTime from apiflask.validators import OneOf -from apps.util import fake +from flask import request +from util import da bp = APIBlueprint("代理管理接口集合", __name__, url_prefix="/agent") @@ -23,19 +25,65 @@ class agent(Schema): start_time = DateTime() +# 数据库列与返回值的键对应关系 +agent_map = { + "AGENT_ID": "id", + "IPADDR": "ipaddr", + "START_TIME": "start_time", + "LAT": "lat", + "LNG": "lng", + "AGENT_TYPE": "atype", + "SYS": "sys", + "PORT": "port", + "CPU_NUM": "cpu_num", + "STATUS": "status", + "IDLE": "idle", + "MEM": "mem" +} + + # 代理注册接口 -@bp.post("/register", ) +@bp.post("/register") @bp.doc("代理注册接口", "返回分配给代理的编号值", hide=True) -@bp.input({ - # 代理类型 - "type": String(required=True, validate=OneOf(["gjst", "mbgz", "ztgz"])), - # 代理的通信端口 - "port": Integer(required=True), - # 代理所在的操作系统 - "sys": String() -}) -def register_agent(query_data): - print(query_data) +# 参数说明 +# 代理类型 +# "type": String(required=True, validate=OneOf(["stgj", "mbgz", "ztgz"])), +# 代理的通信端口 +# "port": Integer(required=True), +# # 代理所在的操作系统 +# "sys": String(), +# # IPv6地址 +# # "v6addr": List(String()), +# # 经度 +# "lat": String(), +# # 纬度 +# "lng": String(), +# # cpu核心数 +# "cpu_num": Integer(), +# # 内存大小 +# "ram_size": Integer(), +# # 已用内存比例 +# "ram_per": Float() +def register_agent(): + data = dict(request.json) + id = "".join(random.sample('1234567890zyxwvutsrqponmlkjihgfedcba', 8)) + # 代理所有参数 + param = {"id": id, + "ipaddr": (request.remote_addr).join(("|" + i) for i in data["v6addr"]), + "start_time": str(datetime.datetime.now()), + "lat": data["lat"], + "lng": data["lng"], + "cpunum": data["cpu_num"], + "mem": data["ram_size"] + "GB" + "(" + {1 - data["ram_per"]} + "%" + "可用)", + "sys": data["sys"], + "status": True, + "idle": True, + "port": data["port"], + "atype": data["atype"], + } + err = insert_agent(param) + if not err: + return {"id": id} # 代理任务下发 @@ -43,7 +91,6 @@ def deliver_task(): pass - @bp.post("/res") @bp.doc("代理输出消息接收接口", hide=True) # TODO:代理输出接收接口实现 @@ -74,19 +121,40 @@ def agent_info(query_data): per_page = query_data["per_page"] page = query_data["page"] agent_list = [] - # TODO:接口数据库贯通实现 - for i in range(per_page): - agent_list.append({ - "id": fake.unique.random_int(), - "ipaddr": [fake.ipv4_public(), fake.ipv6()], - "atype": fake.word(ext_word_list=["参数感知", "状态感知", "攻击渗透"]), - "status": random.choice([True, False]), - "idle": random.choice([True, False]), - "port": random.randint(1000, 65534), - "sys": random.choice(["Linux", "Windows", "Macos"]), - "cpu_num": 4, - "mem": random.choice(["4", "8", "16", "32"]) + "GB(" + str( - '{:.2f}'.format(100 * random.random())) + "%可用)", - "start_time": fake.date_time_between(start_date="-1y")}) - + res = da.get_data(data_type="agent", offset=(page - 1) * per_page, limit=per_page) + for r in res: + agent_list.append({agent_map[key]: value for key, value in r.items()}) return {"agent_data": agent_list} + + +# 代理信息存储到数据库 +def insert_agent(param={}): + sql = """REPLACE INTO %s(AGENT_ID,ADDRv4, + ADDRv6, + CREATED_TIME, + LAT, + LNG , + AGENT_TYPE, + SYS , + PORT , + CPU_NUM , + STATUS , + RAM_SIZE , + WORK_STATUS) VALUES( + %(id)s, + %(v4addr)s, + %(v6addr)s, + %(create_time)s, + %(lat)s, + %(lng)s, + %(atype)s, + %(sys)s, + %(port)s, + %(cpunum)s, + %(status)s, + %(mem)s, + %(work_status)s) + )""" % param + da.cursor.execute(sql) + da.conn.commit() + return None diff --git a/server/apps/util.py b/server/apps/util.py index 9dff80f..ad95002 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -1,8 +1,197 @@ # 工具包 +import pymysql from faker import Faker -from faker.providers import date_time from faker.providers import company +from faker.providers import date_time +from pymysql.cursors import DictCursor + +from settings import * fake = Faker("zh_CN") fake.add_provider(date_time) fake.add_provider(company) + + +# 数据库访问类 +class DataHandler: + # 参数与数据表映射关系 + tabmapping = { + "agent": MYSQL_TAB_AGENT, + "task": MYSQL_TAB_TASK, + "user": MYSQL_TAB_USER} + + # 数据库链接及数据库初始化 + def __init__(self): + # mysql连接,采用字典游标，返回一系列字典值 + self.conn = pymysql.connect(cursorclass=DictCursor, host=MYSQL_HOST, user='root', + password=MYSQL_PAWD, port=MYSQL_PORT) + self.cursor = self.conn.cursor() + + # 初始化sql语句 + # 创建数据库 + dbsql = "CREATE DATABASE IF NOT EXISTS %s" % MYSQL_DATADB + + agentsql = """CREATE TABLE IF NOT EXISTS %s ( + `AGENT_ID` varchar(255) NOT NULL PRIMARY KEY, + `IPADDR` varchar(255),, + `START_TIME` datetime, + `LAT` varchar(255), + `LNG` varchar(255), + `AGENT_TYPE` varchar(255), + `SYS` varchar(255), + `PORT` int, + `CPU_NUM` int, + `STATUS` bool, + `MEM` varchar(255), + `IDLE` bool)ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_AGENT + + policysql = """CREATE TABLE IF NOT EXISTS %s( + `P_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, + `P_EXE` varchar(255), + `P_TYPE` varchar(255), + `P_DESC` varchar(255), + `P_PAYLOAD` varchar(255), + `P_NAME` varchar(255), + `P_PROTO` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_POLICY + + syslogsql = """CREATE TABLE IF NOT EXISTS %s( + `LOG_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, + `LOG_LEVEL` varchar(255), + `LOG_INFO` varchar(255), + `LOG_TIME` datetime, + `S_IP` varchar(255), + `USER_ID` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_SYSLOG + + targetsql = """CREATE TABLE IF NOT EXISTS %s( + `TARGET_ID` varchar(255) NOT NULL PRIMARY KEY, + `ADDRv4` varchar(255), + `ADDRv6` varchar(255), + `IPv6` bool, + `DNSESEC` bool, + `DOT` bool, + `DOH` bool, + `COU` varchar(255), + `ISP` varchar(255), + `LAT` varchar(255), + `LNG` varchar(255), + `UPDATED_TIME` datetime, + `PROTECT` varchar(255), + `DOH_DOMAIN` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_TARGETDATA + + tasksql = """CREATE TABLE IF NOT EXISTS %s( + `TASK_ID` varchar(255) NOT NULL PRIMARY KEY, + `TASK_NAME` varchar(255), + `AGENT_ID` varchar(255), + `CREATED_BY` varchar(255), + `TARGET_IP` varchar(255), + `CREATED_TIME` datetime, + `POLICY` varchar(255), + `STATUS` varchar(255), + `POLICY_DELAY` varchar(255), + `TASK_DELAY` varchar(255), + `TARGET_SCAN` varchar(255), + `TARGET_DOMAIN` varchar(255), + `TARGET_RTYPE` varchar(255), + `TARGET_RR` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_TASK + + tasklogsql = """CREATE TABLE IF NOT EXISTS %s( + `TLOG_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, + `CREATED_BY_AGENT` varchar(255), + `CREATED_TIME` datetime, + `TLOG_LEVEL` varchar(255), + `TLOG_INFO` varchar(255), + `TLOG_TP` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_TASK_LOG + + taskpolicysql = """CREATE TABLE IF NOT EXISTS %s( + `TP_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, + `TP_TIME` datetime, + `POLICY` varchar(255), + `POLICY_PARAM` varchar(255), + `FOR_TASK` varchar(255)) ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_TASKPOLICY + + usersql = """CREATE TABLE IF NOT EXISTS %s( + `USER_ID` varchar(255) NOT NULL PRIMARY KEY, + `USER_NAME` varchar(255), + `CREATED_BY` varchar(255), + `CREATED_TIME` datetime, + `USER_PWD_HASH` varchar(255), + `USER_GROUP` varchar(255)) ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_USER + + # 外键 + fkey1 = """ALTER TABLE %s ADD CONSTRAINT `fk_SYSLOG_USER_1` + FOREIGN KEY(`USER_ID`) REFERENCES %s(`USER_ID`);""" % (MYSQL_TAB_SYSLOG, MYSQL_TAB_USER) + + fkey2 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_AGENT_1` + FOREIGN KEY(`AGENT_ID`) REFERENCES %s(`AGENT_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_AGENT) + + fkey3 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_USER_1` + FOREIGN KEY(`CREATED_BY`) REFERENCES %s(`USER_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_USER) + + fkey4 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_TASK_POLICY_1` + FOREIGN KEY(`POLICY`) REFERENCES %s(`TP_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_TASKPOLICY) + + fkey5 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_LOG_AGENT_1` + FOREIGN KEY(`CREATED_BY_AGENT`) REFERENCES %s(`AGENT_ID`);""" % (MYSQL_TAB_TASK_LOG, MYSQL_TAB_AGENT) + + fkey6 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_LOG_TASK_POLICY_1` + FOREIGN KEY(`TLOG_TP`) REFERENCES %s(`TP_ID`);""" % ( + MYSQL_TAB_TASK_LOG, MYSQL_TAB_TASKPOLICY) + + fkey7 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_POLICY_POLICY_1` + FOREIGN KEY(`POLICY`) REFERENCES %s(`P_ID`);""" % ( + MYSQL_TAB_TASKPOLICY, MYSQL_TAB_POLICY) + + fkey8 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_POLICY_TASK_1` + FOREIGN KEY(`FOR_TASK`) REFERENCES %s(`TASK_ID`);""" % ( + MYSQL_TAB_TASKPOLICY, MYSQL_TAB_TASK) + + # 执行sql语句 + try: + # 创建数据库 + self.cursor.execute(dbsql) + self.conn.commit() + self.conn.select_db(MYSQL_DATADB) + + # 创建表格 + self.cursor.execute(agentsql) + self.cursor.execute(tasksql) + self.cursor.execute(tasklogsql) + self.cursor.execute(targetsql) + self.cursor.execute(syslogsql) + self.cursor.execute(policysql) + self.cursor.execute(taskpolicysql) + self.cursor.execute(usersql) + self.conn.commit() + + # 创建外键 + self.cursor.execute(fkey1) + self.cursor.execute(fkey2) + self.cursor.execute(fkey3) + self.cursor.execute(fkey4) + self.cursor.execute(fkey5) + self.cursor.execute(fkey6) + self.cursor.execute(fkey7) + self.cursor.execute(fkey8) + self.conn.commit() + except Exception as e: + print(e) + + # 获取信息(代理、任务) + # data_type可选范围参照DataHandler.tabmapping的键 + # 若需要按条件检索，则以将检索维度与检索值以字典形式传入search + def get_data(self, data_type="agent", offset=0, limit=10, search=None): + # 参数映射到表名 + tabname = self.tabmapping[data_type] + if search is None: + sql = """SELECT * FROM %s LIMIT %s, %s""" % (tabname, offset, limit) + self.cursor.execute(sql) + return self.cursor.fetchall() + else: + search = dict(search) + key, value = search.popitem() + sql = """SELECT * FROM %s WHERE %s=%s LIMIT %s, %s""" % (tabname, key, value, offset, limit) + self.cursor.execute(sql) + return self.cursor.fetchall() + + +da = DataHandler() diff --git a/server/requirements.txt b/server/requirements.txt index af9f81b..0da63cc 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -6,4 +6,5 @@ numpy~=1.26.3 six~=1.16.0 pandas~=2.1.4 faker~=18.9.0 -dnspython~=2.6.1 \ No newline at end of file +dnspython~=2.6.1 +pymysql~=1.1.0 \ No newline at end of file diff --git a/server/settings.py b/server/settings.py index 5cc1536..4791c0f 100644 --- a/server/settings.py +++ b/server/settings.py @@ -15,4 +15,4 @@ MYSQL_TAB_TARGETDATA = "TARGETDATA" MYSQL_TAB_TASK = "TASK" MYSQL_TAB_POLICY = "POLICY" MYSQL_TAB_TASKPOLICY = "TASK_POLICY" -MYSQL_TASK_LOG = "TASK_LOG" +MYSQL_TAB_TASK_LOG = "TASK_LOG" -- cgit v1.2.3 From 7810aa10cc764db5fe262e0612c664a8df149957 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Tue, 9 Apr 2024 17:17:18 +0800 Subject: 仪表盘界面接口实现 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/sysinfo.py | 31 ++++++++++++++++--------------- server/apps/util.py | 35 ++++++++++++++++++++++++++++++++++- server/settings.py | 3 +++ 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/server/apps/sysinfo.py b/server/apps/sysinfo.py index 3cab7fd..053760c 100644 --- a/server/apps/sysinfo.py +++ b/server/apps/sysinfo.py @@ -1,13 +1,15 @@ # 系统信息接口 import datetime -# 测试用 -from apps.util import fake - from apiflask import APIBlueprint, Schema from apiflask.fields import Date, Integer, Nested, Dict, DateTime, String, List from apiflask.validators import OneOf +# 测试用 +from apps.util import fake +from settings import START_DAY +from util import da + bp = APIBlueprint("仪表盘接口", __name__, url_prefix="/sys") @@ -38,16 +40,16 @@ class LogOut(Schema): def systate(): # TODO:从数据库读取信息并统计 # 已部署代理节点数量 - agent_num = fake.random.randint(10, 100) + agent_num = da.count_data("agent") # 已执行任务数量（包括执行完毕和正在执行） - task_num = fake.random.randint(10, 100) + task_num = da.count_data("task") # 系统已运行天数 - workday = fake.random.randint(10, 100) + workday = datetime.date.today() - START_DAY # 已探测目标统计 - v6dns_num = fake.random.randint(10, 100) - dnssec_num = fake.random.randint(10, 100) - doh_num = fake.random.randint(10, 100) - dot_num = fake.random.randint(10, 100) + v6dns_num = da.count_data("target", {"IPv6": True}) + dnssec_num = da.count_data("target", {"DNSSEC": True}) + doh_num = da.count_data("target", {"DOH": True}) + dot_num = da.count_data("target", {"DOT": True}) # 返回结果 return { "agent_num": agent_num, @@ -73,12 +75,11 @@ def target_date(): day = 1 while day < 8: d = today - datetime.timedelta(day) - # TODO:从数据库读取并统计 dates[d] = { - "v6dns": fake.random.randint(10, 20) * day, - "dnssec": fake.random.randint(10, 20) * day, - "doh": fake.random.randint(10, 20) * day, - "dot": fake.random.randint(10, 20) * day + "v6dns": da.count_data_by_time("target", time=d, search={"IPv6": True}), + "dnssec": da.count_data_by_time("target", time=d, search={"DNSSEC": True}), + "doh": da.count_data_by_time("target", time=d, search={"DOH": True}), + "dot": da.count_data_by_time("target", time=d, search={"DOT": True}) } day += 1 return {"date_data": dates} diff --git a/server/apps/util.py b/server/apps/util.py index ad95002..0b8604b 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -18,7 +18,16 @@ class DataHandler: tabmapping = { "agent": MYSQL_TAB_AGENT, "task": MYSQL_TAB_TASK, - "user": MYSQL_TAB_USER} + "user": MYSQL_TAB_USER, + "target": MYSQL_TAB_TARGETDATA} + + # 参数与数据表中时间字段的映射关系 + timemapping = { + "agent": "START_TIME", + "task": "CREATED_TIME", + "user": "CREATED_TIME", + "target": "UPDATED_TIME", + } # 数据库链接及数据库初始化 def __init__(self): @@ -193,5 +202,29 @@ class DataHandler: self.cursor.execute(sql) return self.cursor.fetchall() + # 统计符合条件的信息数量 + def count_data(self, data_type="agent", search=None): + # 参数映射到表名 + tabname = self.tabmapping[data_type] + if search is None: + sql = """SELECT count(*) FROM %s""" % (tabname) + self.cursor.execute(sql) + return dict(self.cursor.fetchall()[0]).popitem()[1] + else: + search = dict(search) + key, value = search.popitem() + sql = """SELECT count(*) FROM %s WHERE %s=%s""" % (tabname, key, value) + self.cursor.execute(sql) + return dict(self.cursor.fetchall()[0]).popitem()[1] + + def count_data_by_time(self, data_type="agent", time=None, search=None): + tabname = self.tabmapping[data_type] + timename = self.timemapping[data_type] + key, value = search.popitem() + sql = """SELECT COUNT(*) FROM %s WHERE + DATE_FORMAT(%s,'%Y-%m-%d')=%s AND %s=%s""" % (tabname, timename, time, key, value) + self.cursor.execute(sql) + return dict(self.cursor.fetchall()[0]).popitem()[1] + da = DataHandler() diff --git a/server/settings.py b/server/settings.py index 4791c0f..a67923f 100644 --- a/server/settings.py +++ b/server/settings.py @@ -1,5 +1,8 @@ +import datetime + DEBUG = True SERVER_PORT = 12526 +START_DAY = datetime.date.today() # 数据库配置 MYSQL_HOST = '127.0.0.1' -- cgit v1.2.3 From 0fe0e0ec4dda1c78bee1701eb74cc9c4d0128cd9 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Tue, 9 Apr 2024 21:18:21 +0800 Subject: 接口定义完善 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/apps/code/test/README.md | 5 +++++ agent/apps/code/test/hello.py | 1 + agent/apps/script.py | 2 +- server/apps/agentcomm.py | 26 ++++++++++++++++++-------- server/apps/sysinfo.py | 1 - server/apps/sysmange.py | 11 ++++++----- server/apps/target.py | 7 +++++-- server/apps/util.py | 14 +++++++------- 8 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 agent/apps/code/test/README.md diff --git a/agent/apps/code/test/README.md b/agent/apps/code/test/README.md new file mode 100644 index 0000000..2dda2f9 --- /dev/null +++ b/agent/apps/code/test/README.md @@ -0,0 +1,5 @@ +# 简介 + +hello.py 是一个测试文件，用于验证主控端任务下发到代理执行的可行性 + +hello.py接收一个字符串，并返回对应的执行输出 \ No newline at end of file diff --git a/agent/apps/code/test/hello.py b/agent/apps/code/test/hello.py index e8b6f5e..8271b56 100644 --- a/agent/apps/code/test/hello.py +++ b/agent/apps/code/test/hello.py @@ -3,4 +3,5 @@ import argparse parser = argparse.ArgumentParser() parser.add_argument('-s', '--string', type=str, help="输出到控制台的字符串", required=True) args = parser.parse_args() +print("hello.py开始执行，执行参数为" + args.string) print("hello" + str(args.string)) diff --git a/agent/apps/script.py b/agent/apps/script.py index f9886b4..1e2a5f3 100644 --- a/agent/apps/script.py +++ b/agent/apps/script.py @@ -14,7 +14,7 @@ executor = ThreadPoolExecutor(5) @bp.post('/') @bp.doc("渗透任务参数接收接口", "返回任务执行状态") @bp.input({ - 'name': String(required=True), + 'policy': String(required=True), 'param': String(required=True) }) def start_script(query_data): diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index d053c2b..e34a63e 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -1,5 +1,4 @@ # 代理通信与注册接口 -import datetime import random from apiflask import APIBlueprint, Schema @@ -7,6 +6,7 @@ from apiflask.fields import String, Integer, List, Nested, Boolean, DateTime from apiflask.validators import OneOf from flask import request +from settings import * from util import da bp = APIBlueprint("代理管理接口集合", __name__, url_prefix="/agent") @@ -87,25 +87,35 @@ def register_agent(): # 代理任务下发 -def deliver_task(): +def deliver_task(agent_id, policy, policy_param): + # TODO:具体实现 + # 查询agent_id对应代理的ip和端口信息 + # 将policy和policy_param作为参数传递到对应接口 pass @bp.post("/res") @bp.doc("代理输出消息接收接口", hide=True) -# TODO:代理输出接收接口实现 def task_ret(json_data): """ task_ret 代理输出信息接收 Arguments: - json_data -- 请求中的json内容，必须包含"level"和"info"两个keyword + json_data -- 请求中的json内容，必须包含"id"、"taskpolicy"、"level"和"info"两个keyword Returns: - _type_ -- _description_ + "ok" -- 成功执行 """ - print(json_data) - return "ok" + sql = """INSERT INTO %s(CREATED_BY_AGENT,TLOG_LEVEL,TLOG_INFO,TLOG_TP) + VALUES(%s,%s,%s,%s) + """ % (MYSQL_TAB_TASK_LOG, json_data["id"], json_data["level"], json_data["info"], + json_data["taskpolicy"]) + try: + da.cursor.execute(sql) + da.conn.commit() + return "ok" + except Exception as e: + print(e) @bp.get("/") @@ -128,7 +138,7 @@ def agent_info(query_data): # 代理信息存储到数据库 -def insert_agent(param={}): +def insert_agent(param: dict): sql = """REPLACE INTO %s(AGENT_ID,ADDRv4, ADDRv6, CREATED_TIME, diff --git a/server/apps/sysinfo.py b/server/apps/sysinfo.py index 053760c..7aa24f2 100644 --- a/server/apps/sysinfo.py +++ b/server/apps/sysinfo.py @@ -38,7 +38,6 @@ class LogOut(Schema): "target": Nested(TargetOut()) }) def systate(): - # TODO:从数据库读取信息并统计 # 已部署代理节点数量 agent_num = da.count_data("agent") # 已执行任务数量（包括执行完毕和正在执行） diff --git a/server/apps/sysmange.py b/server/apps/sysmange.py index f140506..017abd9 100644 --- a/server/apps/sysmange.py +++ b/server/apps/sysmange.py @@ -1,5 +1,4 @@ # 系统管理——用户管理 -import datetime from apiflask import APIBlueprint, Schema from apiflask.fields import String, Integer, DateTime, List, Nested @@ -27,13 +26,14 @@ class User(Schema): "data": List(Nested(User())) }) def list_user(query_data): - if query_data['page']==1: + # TODO：具体实现 + if query_data['page'] == 1: user_list = [ - {"account": "Root", "username": "管理员", "time": fake.date_time_between(start_date='-10y', end_date='-9y'), - "created_by": "", "group": "Admin"}] + {"account": "Root", "username": "管理员", "time": fake.date_time_between(start_date='-10y', end_date='-9y'), + "created_by": "", "group": "Admin"}] else: user_list = [] - for _ in range(query_data["per_page"]-len(user_list)): + for _ in range(query_data["per_page"] - len(user_list)): user_list.append({ "account": fake.vin(), "username": fake.name(), @@ -53,5 +53,6 @@ def list_user(query_data): "cur_user": String(load_default="Root") }) def create_user(json_data): + # TODO:具体实现 print(json_data) return {"msg": "ok"} diff --git a/server/apps/target.py b/server/apps/target.py index a091173..2388509 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -7,9 +7,10 @@ import pandas as pd import requests from apiflask import APIBlueprint, Schema from apiflask.fields import Integer, String, List, Nested, IP, DateTime -from apps.util import fake from requests.exceptions import Timeout +from apps.util import fake + bp = APIBlueprint("目标信息及状态接口集合", __name__, url_prefix="/target") icmp_delaytable = {} @@ -46,7 +47,7 @@ class Delay(Schema): class DelayOut(Schema): - delay_data = List(Nested(Delay)) + delay_data = List(Nested(Delay())) @bp.get("/delay/") @@ -54,6 +55,7 @@ class DelayOut(Schema): @bp.input({"ip": IP(required=False)}, location="query") @bp.output(DelayOut) def get_pernode_delay(query_data, type): + # TODO:节点选择 addr = "" if 'ip' in query_data.keys(): addr = query_data['ip'] @@ -209,6 +211,7 @@ def record(query_data): "data": List(Nested(Target())) }) def target_info(query_data): + # TODO：具体实现 # 普通检索 if query_data["ip"] is None: per_page = query_data["per_page"] diff --git a/server/apps/util.py b/server/apps/util.py index 0b8604b..1395ebb 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -43,7 +43,7 @@ class DataHandler: agentsql = """CREATE TABLE IF NOT EXISTS %s ( `AGENT_ID` varchar(255) NOT NULL PRIMARY KEY, `IPADDR` varchar(255),, - `START_TIME` datetime, + `START_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `LAT` varchar(255), `LNG` varchar(255), `AGENT_TYPE` varchar(255), @@ -67,7 +67,7 @@ class DataHandler: `LOG_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, `LOG_LEVEL` varchar(255), `LOG_INFO` varchar(255), - `LOG_TIME` datetime, + `LOG_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `S_IP` varchar(255), `USER_ID` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_SYSLOG @@ -83,7 +83,7 @@ class DataHandler: `ISP` varchar(255), `LAT` varchar(255), `LNG` varchar(255), - `UPDATED_TIME` datetime, + `UPDATED_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `PROTECT` varchar(255), `DOH_DOMAIN` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_TARGETDATA @@ -93,7 +93,7 @@ class DataHandler: `AGENT_ID` varchar(255), `CREATED_BY` varchar(255), `TARGET_IP` varchar(255), - `CREATED_TIME` datetime, + `CREATED_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `POLICY` varchar(255), `STATUS` varchar(255), `POLICY_DELAY` varchar(255), @@ -106,14 +106,14 @@ class DataHandler: tasklogsql = """CREATE TABLE IF NOT EXISTS %s( `TLOG_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, `CREATED_BY_AGENT` varchar(255), - `CREATED_TIME` datetime, + `CREATED_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `TLOG_LEVEL` varchar(255), `TLOG_INFO` varchar(255), `TLOG_TP` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_TASK_LOG taskpolicysql = """CREATE TABLE IF NOT EXISTS %s( `TP_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, - `TP_TIME` datetime, + `TP_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `POLICY` varchar(255), `POLICY_PARAM` varchar(255), `FOR_TASK` varchar(255)) ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_TASKPOLICY @@ -122,7 +122,7 @@ class DataHandler: `USER_ID` varchar(255) NOT NULL PRIMARY KEY, `USER_NAME` varchar(255), `CREATED_BY` varchar(255), - `CREATED_TIME` datetime, + `CREATED_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `USER_PWD_HASH` varchar(255), `USER_GROUP` varchar(255)) ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_USER -- cgit v1.2.3 From 8f38b07715fca9f698666d82e11d81de31be159a Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Wed, 10 Apr 2024 16:22:53 +0800 Subject: 仪表盘界面接口开发尝试 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/sysinfo.py | 37 ++++++++++++++++++++++++------------- server/apps/util.py | 3 ++- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/server/apps/sysinfo.py b/server/apps/sysinfo.py index 7aa24f2..bae76b2 100644 --- a/server/apps/sysinfo.py +++ b/server/apps/sysinfo.py @@ -1,17 +1,22 @@ # 系统信息接口 -import datetime from apiflask import APIBlueprint, Schema from apiflask.fields import Date, Integer, Nested, Dict, DateTime, String, List from apiflask.validators import OneOf # 测试用 -from apps.util import fake -from settings import START_DAY +from settings import * from util import da bp = APIBlueprint("仪表盘接口", __name__, url_prefix="/sys") +log_map = { + "LOG_LEVEL": "level", + "LOG_INFO": "info", + "LOG_TIME": "time", + "S_IP": "ip", +} + class TargetOut(Schema): v6dns = Integer() @@ -84,6 +89,7 @@ def target_date(): return {"date_data": dates} +# 系统操作日志获取接口 @bp.get("/log") @bp.input({ "per_page": Integer(load_default=10), @@ -98,14 +104,19 @@ def sys_log(query_data): per_page = query_data["per_page"] # 页数 page = query_data["page"] - log_list = [] - # TODO:具体实现 - for i in range(per_page): - log_list.append({ - "time": fake.date_time_between(start_date="-1y"), - "level": fake.word(ext_word_list=["INFO", "WARNING", "ERROR"]), - "info": fake.text(max_nb_chars=20, ext_word_list=None), - "user": fake.name(), - "ip": fake.ipv4() - }) + tabs = { + "syslog": MYSQL_TAB_SYSLOG, + "user": MYSQL_TAB_USER, + "start": (page - 1) * per_page, + "stop": per_page + } + sql = """SELECT + s.LOG_LEVEL as level, + s.LOG_INFO as info, + s.LOG_TIME as time, + s.S_IP as ip, + CONCAT(u.USER_NAME,"(ID:",u.USER_ID,")") as user + from %(syslog)s s,%(user)s u WHERE s.USER_ID=u.USER_ID LIMIT %(start)s,%(stop)s """ % tabs + da.cursor.execute(sql) + log_list = da.cursor.fetchall() return {"log_data": log_list} diff --git a/server/apps/util.py b/server/apps/util.py index 1395ebb..fa5caae 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -19,7 +19,8 @@ class DataHandler: "agent": MYSQL_TAB_AGENT, "task": MYSQL_TAB_TASK, "user": MYSQL_TAB_USER, - "target": MYSQL_TAB_TARGETDATA} + "target": MYSQL_TAB_TARGETDATA, + "syslog": MYSQL_TAB_SYSLOG, } # 参数与数据表中时间字段的映射关系 timemapping = { -- cgit v1.2.3 From eb4ce6f691cdd188d5ca9c39e5f005fe8b99a96b Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Wed, 10 Apr 2024 16:23:34 +0800 Subject: 新增代理服务类型参数映射 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/agent/app.py b/agent/app.py index ad85985..15d0773 100644 --- a/agent/app.py +++ b/agent/app.py @@ -17,6 +17,12 @@ app = APIFlask(__name__, template_folder='./static/templates') app.register_blueprint(scriptbp) # 代理配置 config = {} +# 代理类型参数映射表 +atype_map = { + "stgj": "渗透攻击", + "csgz": "参数感知", + "ztgz": "状态感知" +} @app.get('/') @@ -64,7 +70,7 @@ def nodeinfo(): def registernode(proto="http", port=2525, atype="stgj", server="127.0.0.1:8888"): info = nodeinfo() info["port"] = port - info["atype"] = atype + info["atype"] = atype_map[atype] print(info) r = requests.post(proto + "://" + server + "/agent/register", json=info) if r.status_code != 200: @@ -87,7 +93,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("-p", "--port", type=int, default=2525, help="代理的开放通信端口") parser.add_argument("-t", "--atype", type=str, default="stgj", - help="代理的工作类型 {stgj(渗透攻击) / mbgz(目标感知) / ztgz(状态感知)}") + help="代理的工作类型 {stgj(渗透攻击) / csgz(参数感知) / ztgz(状态感知)}") parser.add_argument("-s", "--server", type=str, default="127.0.0.1:8888", help="主控端访问地址+端口号") # 解析参数 -- cgit v1.2.3 From 5f8a190f889f1aab771f8662373ddc810ddfeb7f Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 11 Apr 2024 15:34:38 +0800 Subject: 新增日志记录功能 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/util.py | 15 +++++++++++++++ server/requirements.txt | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/server/apps/util.py b/server/apps/util.py index fa5caae..025d072 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -3,8 +3,22 @@ import pymysql from faker import Faker from faker.providers import company from faker.providers import date_time +from loguru import logger from pymysql.cursors import DictCursor + +def debug(message, *args, **kwargs): + logger.debug(message, *args, **kwargs) + + +def info(message, *args, **kwargs): + logger.info(message, *args, **kwargs) + + +def error(message, *args, **kwargs): + logger.error(message, *args, **kwargs) + + from settings import * fake = Faker("zh_CN") @@ -32,6 +46,7 @@ class DataHandler: # 数据库链接及数据库初始化 def __init__(self): + # mysql连接,采用字典游标，返回一系列字典值 self.conn = pymysql.connect(cursorclass=DictCursor, host=MYSQL_HOST, user='root', password=MYSQL_PAWD, port=MYSQL_PORT) diff --git a/server/requirements.txt b/server/requirements.txt index 0da63cc..56babab 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -7,4 +7,5 @@ six~=1.16.0 pandas~=2.1.4 faker~=18.9.0 dnspython~=2.6.1 -pymysql~=1.1.0 \ No newline at end of file +pymysql~=1.1.0 +loguru~=0.5.3 \ No newline at end of file -- cgit v1.2.3 From 2318b942d556c9e94d589a6b989c6487f9991a5e Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 11 Apr 2024 15:35:40 +0800 Subject: 目标信息获取接口开发 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/target.py | 80 ++++++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/server/apps/target.py b/server/apps/target.py index 2388509..d3e1ab1 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -1,15 +1,14 @@ # 目标状态感知 -import random # 时延测试接口 import threading import pandas as pd import requests from apiflask import APIBlueprint, Schema -from apiflask.fields import Integer, String, List, Nested, IP, DateTime +from apiflask.fields import Integer, String, List, Nested, IP, DateTime, Dict from requests.exceptions import Timeout -from apps.util import fake +from util import da bp = APIBlueprint("目标信息及状态接口集合", __name__, url_prefix="/target") @@ -17,6 +16,16 @@ icmp_delaytable = {} tcp_delaytable = {} dns_delaytable = {} +target_map = { + "ADDRv4": "ipv4", + "ADDRv6": "ipv6", + "COU": "cou", + "ISP": "isp", + "LAT": "lat", + "LNG": "lng", + "UPDATED_TIME": "time" +} + class Target(Schema): ipv4 = String() @@ -154,7 +163,7 @@ def dns_delay_query(target, addr): # 状态感知——DNS记录测试接口 import dns.nameserver from apiflask.fields import String -from apiflask.validators import OneOf +from apiflask.validators import OneOf, ContainsOnly from dns import resolver @@ -165,6 +174,9 @@ from dns import resolver 'domain': String(required=True), 'qtype': String(required=True, validate=OneOf(['A', 'AAAA', "CNAME", "NS"])) }, location='query') +@bp.output({ + "ans": List(Dict(String(validate=ContainsOnly(["rrset"])), String())) +}) def record(query_data): # 特殊协议头 protols = ["https", "tls"] @@ -211,37 +223,33 @@ def record(query_data): "data": List(Nested(Target())) }) def target_info(query_data): - # TODO：具体实现 - # 普通检索 + per_page = query_data["per_page"] + page = query_data["page"] + target_list = [] + if query_data["ip"] is None: - per_page = query_data["per_page"] - target_list = [] - - for _ in range(per_page): - lat, lng, _, cou, _ = fake.location_on_land() - target_list.append({ - "ipv4": fake.ipv4_public(), - "ipv6": fake.ipv6(), - "protocol": random.choices(["IPv6", "DNSSEC", "DoH", "DoT"], k=random.randint(1, 4)), - "protect": random.choices(["0x20", "端口随机化", "请求最小化", "DNSSEC验证"], k=random.randint(1, 4)), - "cou": cou, - "lat": lat, - "lng": lng, - "isp": fake.company(), - "time": fake.date_time_between('-2y'), - }) - return {"data": target_list} - # 查询目标 + # 普通检索 + res = da.get_data(data_type="target", offset=(page - 1) * per_page, limit=per_page) else: - lat, lng, _, cou, _ = fake.location_on_land() - target = [{"ipv4": query_data["ip"], - "ipv6": fake.ipv6(), - "protocol": random.choices(["IPv6", "DNSSEC", "DoH", "DoT"], k=random.randint(1, 4)), - "protect": random.choices(["0x20", "端口随机化", "请求最小化", "DNSSEC验证"], - k=random.randint(1, 4)), - "cou": cou, - "lat": lat, - "lng": lng, - "isp": fake.company(), - "time": fake.date_time_between('-2y'), }] - return {"data": target} + # 查询目标 + res = da.get_data(data_type="target", offset=(page - 1) * per_page, limit=per_page, + search={"ip": query_data["ip"]}) + + # 结果转换 + for r in res: + target = {} + # 支持的协议特殊处理 + protocol = [] + for k, v in r.items(): + if k in ["IPv6", "DNSSEC", "DOH", "DOT"] and v is True: + protocol.append(k) + target["protocol"] = protocol + # 防护措施特殊处理 + protect = str(r["PROTECT"]).split("|") + target["protect"] = protect + # 其他统一处理 + for k in target_map.keys(): + # 原数据的值赋给经过映射之后的键 + target[target_map[k]] = r[k] + target_list.append(target) + return {"data": target_list} -- cgit v1.2.3 From 971804d3503ae151a0528aabcc871028c788594c Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 11 Apr 2024 17:02:17 +0800 Subject: 数据库初始化功能实现 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/util.py | 128 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 80 insertions(+), 48 deletions(-) diff --git a/server/apps/util.py b/server/apps/util.py index 025d072..3ef9063 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -58,7 +58,7 @@ class DataHandler: agentsql = """CREATE TABLE IF NOT EXISTS %s ( `AGENT_ID` varchar(255) NOT NULL PRIMARY KEY, - `IPADDR` varchar(255),, + `IPADDR` varchar(255), `START_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `LAT` varchar(255), `LNG` varchar(255), @@ -71,7 +71,7 @@ class DataHandler: `IDLE` bool)ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_AGENT policysql = """CREATE TABLE IF NOT EXISTS %s( - `P_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, + `P_ID` varchar(255) NOT NULL PRIMARY KEY, `P_EXE` varchar(255), `P_TYPE` varchar(255), `P_DESC` varchar(255), @@ -85,7 +85,9 @@ class DataHandler: `LOG_INFO` varchar(255), `LOG_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `S_IP` varchar(255), - `USER_ID` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_SYSLOG + `USER_ID` varchar(255), + FOREIGN KEY(`USER_ID`) REFERENCES %s(`USER_ID`) + )ENGINE=innodb DEFAULT CHARSET=utf8; """ % (MYSQL_TAB_SYSLOG, MYSQL_TAB_USER) targetsql = """CREATE TABLE IF NOT EXISTS %s( `TARGET_ID` varchar(255) NOT NULL PRIMARY KEY, @@ -110,14 +112,18 @@ class DataHandler: `CREATED_BY` varchar(255), `TARGET_IP` varchar(255), `CREATED_TIME` datetime DEFAULT CURRENT_TIMESTAMP, - `POLICY` varchar(255), + `POLICY` int, `STATUS` varchar(255), `POLICY_DELAY` varchar(255), `TASK_DELAY` varchar(255), `TARGET_SCAN` varchar(255), `TARGET_DOMAIN` varchar(255), `TARGET_RTYPE` varchar(255), - `TARGET_RR` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8; """ % MYSQL_TAB_TASK + `TARGET_RR` varchar(255), + FOREIGN KEY(`AGENT_ID`) REFERENCES %s(`AGENT_ID`), + FOREIGN KEY(`CREATED_BY`) REFERENCES %s(`USER_ID`) + )ENGINE=innodb DEFAULT CHARSET=utf8; """ % ( + MYSQL_TAB_TASK, MYSQL_TAB_AGENT, MYSQL_TAB_USER) tasklogsql = """CREATE TABLE IF NOT EXISTS %s( `TLOG_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, @@ -125,14 +131,20 @@ class DataHandler: `CREATED_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `TLOG_LEVEL` varchar(255), `TLOG_INFO` varchar(255), - `TLOG_TP` varchar(255))ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_TASK_LOG + `TLOG_TP` int, + FOREIGN KEY(`CREATED_BY_AGENT`) REFERENCES %s(`AGENT_ID`), + FOREIGN KEY(`TLOG_TP`) REFERENCES %s(`TP_ID`) + )ENGINE=innodb DEFAULT CHARSET=utf8;""" % (MYSQL_TAB_TASK_LOG, MYSQL_TAB_AGENT, MYSQL_TAB_TASKPOLICY) taskpolicysql = """CREATE TABLE IF NOT EXISTS %s( `TP_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, `TP_TIME` datetime DEFAULT CURRENT_TIMESTAMP, `POLICY` varchar(255), `POLICY_PARAM` varchar(255), - `FOR_TASK` varchar(255)) ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_TASKPOLICY + `FOR_TASK` varchar(255), + FOREIGN KEY(`POLICY`) REFERENCES %s(`P_ID`), + FOREIGN KEY(`FOR_TASK`) REFERENCES %s(`TASK_ID`) + ) ENGINE=innodb DEFAULT CHARSET=utf8;""" % (MYSQL_TAB_TASKPOLICY, MYSQL_TAB_POLICY, MYSQL_TAB_TASK) usersql = """CREATE TABLE IF NOT EXISTS %s( `USER_ID` varchar(255) NOT NULL PRIMARY KEY, @@ -142,33 +154,51 @@ class DataHandler: `USER_PWD_HASH` varchar(255), `USER_GROUP` varchar(255)) ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_USER - # 外键 - fkey1 = """ALTER TABLE %s ADD CONSTRAINT `fk_SYSLOG_USER_1` - FOREIGN KEY(`USER_ID`) REFERENCES %s(`USER_ID`);""" % (MYSQL_TAB_SYSLOG, MYSQL_TAB_USER) - - fkey2 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_AGENT_1` - FOREIGN KEY(`AGENT_ID`) REFERENCES %s(`AGENT_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_AGENT) - - fkey3 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_USER_1` - FOREIGN KEY(`CREATED_BY`) REFERENCES %s(`USER_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_USER) - - fkey4 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_TASK_POLICY_1` - FOREIGN KEY(`POLICY`) REFERENCES %s(`TP_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_TASKPOLICY) - - fkey5 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_LOG_AGENT_1` - FOREIGN KEY(`CREATED_BY_AGENT`) REFERENCES %s(`AGENT_ID`);""" % (MYSQL_TAB_TASK_LOG, MYSQL_TAB_AGENT) - - fkey6 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_LOG_TASK_POLICY_1` - FOREIGN KEY(`TLOG_TP`) REFERENCES %s(`TP_ID`);""" % ( - MYSQL_TAB_TASK_LOG, MYSQL_TAB_TASKPOLICY) - - fkey7 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_POLICY_POLICY_1` - FOREIGN KEY(`POLICY`) REFERENCES %s(`P_ID`);""" % ( - MYSQL_TAB_TASKPOLICY, MYSQL_TAB_POLICY) - - fkey8 = """ALTER TABLE %s ADD CONSTRAINT `fk_TASK_POLICY_TASK_1` - FOREIGN KEY(`FOR_TASK`) REFERENCES %s(`TASK_ID`);""" % ( - MYSQL_TAB_TASKPOLICY, MYSQL_TAB_TASK) + # 外键，考虑重复外键的处理 + # fkey1 = """ + # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_SYSLOG_USER_1`; + # ALTER TABLE %s ADD CONSTRAINT `fk_SYSLOG_USER_1` + # FOREIGN KEY(`USER_ID`) REFERENCES %s(`USER_ID`);""" % (MYSQL_TAB_SYSLOG, MYSQL_TAB_SYSLOG, MYSQL_TAB_USER) + + # fkey2 = """ + # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_AGENT_1`; + # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_AGENT_1` + # FOREIGN KEY(`AGENT_ID`) REFERENCES %s(`AGENT_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_TASK, MYSQL_TAB_AGENT) + + # fkey3 = """ + # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_USER_1`; + # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_USER_1` + # FOREIGN KEY(`CREATED_BY`) REFERENCES %s(`USER_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_TASK, MYSQL_TAB_USER) + + # 和task_policysql存在双向外键引用的问题，需要额外定义 + fkey = """ + ALTER TABLE %s ADD CONSTRAINT `fk_TASK_TASK_POLICY_1` + FOREIGN KEY(`POLICY`) REFERENCES %s(`TP_ID`)""" % ( + MYSQL_TAB_TASK, MYSQL_TAB_TASKPOLICY) + + # fkey5 = """ + # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_LOG_AGENT_1`; + # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_LOG_AGENT_1` + # FOREIGN KEY(`CREATED_BY_AGENT`) REFERENCES %s(`AGENT_ID`);""" % ( + # MYSQL_TAB_TASK_LOG, MYSQL_TAB_TASK_LOG, MYSQL_TAB_AGENT) + + # fkey6 = """ + # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_LOG_TASK_POLICY_1`; + # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_LOG_TASK_POLICY_1` + # FOREIGN KEY(`TLOG_TP`) REFERENCES %s(`TP_ID`);""" % ( + # MYSQL_TAB_TASK_LOG, MYSQL_TAB_TASK_LOG, MYSQL_TAB_TASKPOLICY) + + # fkey7 = """ + # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_POLICY_POLICY_1`; + # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_POLICY_POLICY_1` + # FOREIGN KEY(`POLICY`) REFERENCES %s(`P_ID`);""" % ( + # MYSQL_TAB_TASKPOLICY, MYSQL_TAB_TASKPOLICY, MYSQL_TAB_POLICY) + + # fkey8 = """ + # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_POLICY_TASK_1`; + # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_POLICY_TASK_1` + # FOREIGN KEY(`FOR_TASK`) REFERENCES %s(`TASK_ID`);""" % ( + # MYSQL_TAB_TASKPOLICY, MYSQL_TAB_TASKPOLICY, MYSQL_TAB_TASK) # 执行sql语句 try: @@ -179,27 +209,29 @@ class DataHandler: # 创建表格 self.cursor.execute(agentsql) - self.cursor.execute(tasksql) - self.cursor.execute(tasklogsql) - self.cursor.execute(targetsql) + self.cursor.execute(usersql) self.cursor.execute(syslogsql) + self.cursor.execute(targetsql) + self.cursor.execute(tasksql) self.cursor.execute(policysql) self.cursor.execute(taskpolicysql) - self.cursor.execute(usersql) + self.cursor.execute(tasklogsql) self.conn.commit() # 创建外键 - self.cursor.execute(fkey1) - self.cursor.execute(fkey2) - self.cursor.execute(fkey3) - self.cursor.execute(fkey4) - self.cursor.execute(fkey5) - self.cursor.execute(fkey6) - self.cursor.execute(fkey7) - self.cursor.execute(fkey8) - self.conn.commit() + self.cursor.execute(fkey) + # self.cursor.execute(fkey2) + # self.cursor.execute(fkey3) + # self.cursor.execute(fkey4) + # self.cursor.execute(fkey5) + # self.cursor.execute(fkey6) + # self.cursor.execute(fkey7) + # self.cursor.execute(fkey8) + # self.conn.commit() except Exception as e: - print(e) + # 重复外键 + if str(e).find("(1826"): + pass # 获取信息(代理、任务) # data_type可选范围参照DataHandler.tabmapping的键 -- cgit v1.2.3 From 52a76605ca8180c7cacfb7a2911e47a3c1eafbde Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 11 Apr 2024 17:03:11 +0800 Subject: 删除废弃代码 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/util.py | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) diff --git a/server/apps/util.py b/server/apps/util.py index 3ef9063..67227e4 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -154,52 +154,12 @@ class DataHandler: `USER_PWD_HASH` varchar(255), `USER_GROUP` varchar(255)) ENGINE=innodb DEFAULT CHARSET=utf8;""" % MYSQL_TAB_USER - # 外键，考虑重复外键的处理 - # fkey1 = """ - # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_SYSLOG_USER_1`; - # ALTER TABLE %s ADD CONSTRAINT `fk_SYSLOG_USER_1` - # FOREIGN KEY(`USER_ID`) REFERENCES %s(`USER_ID`);""" % (MYSQL_TAB_SYSLOG, MYSQL_TAB_SYSLOG, MYSQL_TAB_USER) - - # fkey2 = """ - # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_AGENT_1`; - # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_AGENT_1` - # FOREIGN KEY(`AGENT_ID`) REFERENCES %s(`AGENT_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_TASK, MYSQL_TAB_AGENT) - - # fkey3 = """ - # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_USER_1`; - # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_USER_1` - # FOREIGN KEY(`CREATED_BY`) REFERENCES %s(`USER_ID`);""" % (MYSQL_TAB_TASK, MYSQL_TAB_TASK, MYSQL_TAB_USER) - # 和task_policysql存在双向外键引用的问题，需要额外定义 fkey = """ ALTER TABLE %s ADD CONSTRAINT `fk_TASK_TASK_POLICY_1` FOREIGN KEY(`POLICY`) REFERENCES %s(`TP_ID`)""" % ( MYSQL_TAB_TASK, MYSQL_TAB_TASKPOLICY) - # fkey5 = """ - # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_LOG_AGENT_1`; - # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_LOG_AGENT_1` - # FOREIGN KEY(`CREATED_BY_AGENT`) REFERENCES %s(`AGENT_ID`);""" % ( - # MYSQL_TAB_TASK_LOG, MYSQL_TAB_TASK_LOG, MYSQL_TAB_AGENT) - - # fkey6 = """ - # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_LOG_TASK_POLICY_1`; - # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_LOG_TASK_POLICY_1` - # FOREIGN KEY(`TLOG_TP`) REFERENCES %s(`TP_ID`);""" % ( - # MYSQL_TAB_TASK_LOG, MYSQL_TAB_TASK_LOG, MYSQL_TAB_TASKPOLICY) - - # fkey7 = """ - # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_POLICY_POLICY_1`; - # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_POLICY_POLICY_1` - # FOREIGN KEY(`POLICY`) REFERENCES %s(`P_ID`);""" % ( - # MYSQL_TAB_TASKPOLICY, MYSQL_TAB_TASKPOLICY, MYSQL_TAB_POLICY) - - # fkey8 = """ - # ALTER TABLE %s DROP FOREIGN KEY IF EXISTS `fk_TASK_POLICY_TASK_1`; - # ALTER TABLE %s ADD CONSTRAINT `fk_TASK_POLICY_TASK_1` - # FOREIGN KEY(`FOR_TASK`) REFERENCES %s(`TASK_ID`);""" % ( - # MYSQL_TAB_TASKPOLICY, MYSQL_TAB_TASKPOLICY, MYSQL_TAB_TASK) - # 执行sql语句 try: # 创建数据库 @@ -220,14 +180,6 @@ class DataHandler: # 创建外键 self.cursor.execute(fkey) - # self.cursor.execute(fkey2) - # self.cursor.execute(fkey3) - # self.cursor.execute(fkey4) - # self.cursor.execute(fkey5) - # self.cursor.execute(fkey6) - # self.cursor.execute(fkey7) - # self.cursor.execute(fkey8) - # self.conn.commit() except Exception as e: # 重复外键 if str(e).find("(1826"): -- cgit v1.2.3 From b821f6195c171ddab3c3ee8de501168d4e0dfaac Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Fri, 12 Apr 2024 15:52:26 +0800 Subject: 1. util模块问题修复 2. 数据库连接失败处理逻辑完善 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 2 +- server/apps/sysinfo.py | 2 +- server/apps/sysmange.py | 2 +- server/apps/target.py | 2 +- server/apps/task.py | 2 +- server/apps/util.py | 11 +++++++++-- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index e34a63e..9ff2eb4 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -7,7 +7,7 @@ from apiflask.validators import OneOf from flask import request from settings import * -from util import da +from .util import da bp = APIBlueprint("代理管理接口集合", __name__, url_prefix="/agent") diff --git a/server/apps/sysinfo.py b/server/apps/sysinfo.py index bae76b2..b18cb65 100644 --- a/server/apps/sysinfo.py +++ b/server/apps/sysinfo.py @@ -6,7 +6,7 @@ from apiflask.validators import OneOf # 测试用 from settings import * -from util import da +from .util import da bp = APIBlueprint("仪表盘接口", __name__, url_prefix="/sys") diff --git a/server/apps/sysmange.py b/server/apps/sysmange.py index 017abd9..9f391c7 100644 --- a/server/apps/sysmange.py +++ b/server/apps/sysmange.py @@ -3,7 +3,7 @@ from apiflask import APIBlueprint, Schema from apiflask.fields import String, Integer, DateTime, List, Nested -from apps.util import fake +from .util import fake bp = APIBlueprint("系统管理接口集合", __name__, url_prefix="/user") diff --git a/server/apps/target.py b/server/apps/target.py index d3e1ab1..a0606a4 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -8,7 +8,7 @@ from apiflask import APIBlueprint, Schema from apiflask.fields import Integer, String, List, Nested, IP, DateTime, Dict from requests.exceptions import Timeout -from util import da +from .util import da bp = APIBlueprint("目标信息及状态接口集合", __name__, url_prefix="/target") diff --git a/server/apps/task.py b/server/apps/task.py index 3905df4..b5edae1 100644 --- a/server/apps/task.py +++ b/server/apps/task.py @@ -6,7 +6,7 @@ from apiflask import APIBlueprint, Schema from apiflask.fields import String, Integer, IP, DateTime, List, Nested from apiflask.validators import OneOf -from apps.util import fake +from .util import fake bp = APIBlueprint("任务管理接口集合", __name__, url_prefix="/task") diff --git a/server/apps/util.py b/server/apps/util.py index 67227e4..7c0c99a 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -1,4 +1,6 @@ # 工具包 +import sys + import pymysql from faker import Faker from faker.providers import company @@ -48,8 +50,13 @@ class DataHandler: def __init__(self): # mysql连接,采用字典游标，返回一系列字典值 - self.conn = pymysql.connect(cursorclass=DictCursor, host=MYSQL_HOST, user='root', - password=MYSQL_PAWD, port=MYSQL_PORT) + try: + self.conn = pymysql.connect(cursorclass=DictCursor, host=MYSQL_HOST, user='root', + password=MYSQL_PAWD, port=MYSQL_PORT) + except Exception as e: + error(str(e)) + # 数据库连接失败，停止后续操作 + sys.exit(1) self.cursor = self.conn.cursor() # 初始化sql语句 -- cgit v1.2.3 From a176da2b9ccb68aa346e84417f57bfa3ff072fa2 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Fri, 12 Apr 2024 16:10:03 +0800 Subject: 1. 代理信息获取接口初步测试完成 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 9ff2eb4..8dd78d3 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -7,7 +7,7 @@ from apiflask.validators import OneOf from flask import request from settings import * -from .util import da +from .util import da, debug bp = APIBlueprint("代理管理接口集合", __name__, url_prefix="/agent") @@ -132,15 +132,18 @@ def agent_info(query_data): page = query_data["page"] agent_list = [] res = da.get_data(data_type="agent", offset=(page - 1) * per_page, limit=per_page) + debug(res) for r in res: - agent_list.append({agent_map[key]: value for key, value in r.items()}) + agent = {} + for key, value in r.items(): + agent[agent_map[key]] = value if key != "IPADDR" else str(value).split("|") + agent_list.append(agent) return {"agent_data": agent_list} # 代理信息存储到数据库 def insert_agent(param: dict): - sql = """REPLACE INTO %s(AGENT_ID,ADDRv4, - ADDRv6, + sql = """REPLACE INTO %s(AGENT_ID,IPADDR, CREATED_TIME, LAT, LNG , @@ -152,8 +155,7 @@ def insert_agent(param: dict): RAM_SIZE , WORK_STATUS) VALUES( %(id)s, - %(v4addr)s, - %(v6addr)s, + %(ipaddr)s, %(create_time)s, %(lat)s, %(lng)s, -- cgit v1.2.3 From 8c6d7e3dc8a29d67dcf0638aac8b22d24eaa9eea Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Fri, 12 Apr 2024 16:14:16 +0800 Subject: 1. 代理信息获取接口初步测试完成 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 8dd78d3..0047c4d 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -119,7 +119,7 @@ def task_ret(json_data): @bp.get("/") -@bp.doc("代理信息获取接口") +@bp.doc("代理信息获取接口", description="已测试") @bp.input({ "page": Integer(load_default=1), "per_page": Integer(load_default=10) -- cgit v1.2.3 From 914a569750942e61af3bebbbf99e834d35a66200 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Fri, 12 Apr 2024 16:30:46 +0800 Subject: 1. 目标信息获取接口初步测试完成 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 3 +-- server/apps/target.py | 5 +++-- server/apps/util.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 0047c4d..178f66e 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -7,7 +7,7 @@ from apiflask.validators import OneOf from flask import request from settings import * -from .util import da, debug +from .util import da bp = APIBlueprint("代理管理接口集合", __name__, url_prefix="/agent") @@ -132,7 +132,6 @@ def agent_info(query_data): page = query_data["page"] agent_list = [] res = da.get_data(data_type="agent", offset=(page - 1) * per_page, limit=per_page) - debug(res) for r in res: agent = {} for key, value in r.items(): diff --git a/server/apps/target.py b/server/apps/target.py index a0606a4..6ea43d1 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -8,7 +8,7 @@ from apiflask import APIBlueprint, Schema from apiflask.fields import Integer, String, List, Nested, IP, DateTime, Dict from requests.exceptions import Timeout -from .util import da +from .util import da, debug bp = APIBlueprint("目标信息及状态接口集合", __name__, url_prefix="/target") @@ -241,7 +241,8 @@ def target_info(query_data): # 支持的协议特殊处理 protocol = [] for k, v in r.items(): - if k in ["IPv6", "DNSSEC", "DOH", "DOT"] and v is True: + debug(k + ":" + str(v)) + if k in ["IPv6", "DNSSEC", "DOH", "DOT"] and bool(v) is True: protocol.append(k) target["protocol"] = protocol # 防护措施特殊处理 diff --git a/server/apps/util.py b/server/apps/util.py index 7c0c99a..c789f61 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -97,11 +97,11 @@ class DataHandler: )ENGINE=innodb DEFAULT CHARSET=utf8; """ % (MYSQL_TAB_SYSLOG, MYSQL_TAB_USER) targetsql = """CREATE TABLE IF NOT EXISTS %s( - `TARGET_ID` varchar(255) NOT NULL PRIMARY KEY, + `TARGET_ID` int NOT NULL AUTO_INCREMENT PRIMARY KEY, `ADDRv4` varchar(255), `ADDRv6` varchar(255), `IPv6` bool, - `DNSESEC` bool, + `DNSSEC` bool, `DOT` bool, `DOH` bool, `COU` varchar(255), -- cgit v1.2.3 From 9030bae9cafec72f36b1a770663a4b3d3057b0aa Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Fri, 12 Apr 2024 16:55:31 +0800 Subject: 新增跨域支持 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/app.py | 3 +++ server/apps/target.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/app.py b/server/app.py index 4f02203..b1ed6a5 100644 --- a/server/app.py +++ b/server/app.py @@ -1,4 +1,5 @@ from apiflask import APIFlask +from flask_cors import CORS import settings from apps.agentcomm import bp as agentbp @@ -9,6 +10,8 @@ from apps.task import bp as taskbp # 注册蓝图 app = APIFlask(__name__, template_folder='./static/templates') +# 跨域支持 +CORS(app) # 目标状态获取接口 app.register_blueprint(targetbp) # 代理接口 diff --git a/server/apps/target.py b/server/apps/target.py index 6ea43d1..98a5294 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -213,7 +213,7 @@ def record(query_data): @bp.get("/") -@bp.doc("目标信息获取接口", "返回目标信息") +@bp.doc("目标信息获取接口", "返回目标信息，已测试") @bp.input({ "per_page": Integer(load_default=10), "page": Integer(load_default=1), -- cgit v1.2.3 From 1286fbdfcf7b71f8962b531abe759e4ab9b2d2aa Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Tue, 16 Apr 2024 18:15:09 +0800 Subject: 接口定义完善 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 178f66e..4958d57 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -11,11 +11,14 @@ from .util import da bp = APIBlueprint("代理管理接口集合", __name__, url_prefix="/agent") +# 代理类型 +agent_type = ["渗透攻击", "状态感知", "参数感知"] + class agent(Schema): id = String() ipaddr = List(String()) - atype = String(validate=OneOf(["渗透攻击", "状态感知", "参数感知"])) + atype = String(validate=OneOf(agent_type)) status = Boolean() idle = Boolean() port = Integer() @@ -119,14 +122,23 @@ def task_ret(json_data): @bp.get("/") -@bp.doc("代理信息获取接口", description="已测试") +@bp.doc("代理信息获取接口", description="输入参数说明：
" + + "atype:能力类型，可选参数范围及对应含义为：all-全部（默认），gjst-攻击渗透，ztgz-状态感知，csgz-参数感知
" + + "status:当前状态，可选参数范围及对应含义为：2-全部(默认)，1-在线，0-离线
" + + "idle:是否空闲，可选参数范围及对应含义为：2-全部(默认)，1-空闲，0-执行中
") @bp.input({ + "atype": String(load_default="all"), + "status": Integer(validate=OneOf([0, 1, 2])), + "idle": Integer(validate=OneOf([0, 1, 2])), "page": Integer(load_default=1), "per_page": Integer(load_default=10) }, location="query") @bp.output({ - "agent_data": List(Nested(agent())) + "code": Integer(), + "agent_data": List(Nested(agent())), + "total": Integer() }) +# TODO: code total def agent_info(query_data): per_page = query_data["per_page"] page = query_data["page"] @@ -137,7 +149,7 @@ def agent_info(query_data): for key, value in r.items(): agent[agent_map[key]] = value if key != "IPADDR" else str(value).split("|") agent_list.append(agent) - return {"agent_data": agent_list} + return {"code": 200, "agent_data": agent_list, "total": 100} # 代理信息存储到数据库 @@ -169,3 +181,19 @@ def insert_agent(param: dict): da.cursor.execute(sql) da.conn.commit() return None + + +@bp.doc("代理删除接口", "输入参数说明：
" + + "id: 代理编号") +@bp.post("/del") +@bp.input({ + "id": String(required=True) +}) +@bp.output({ + "code": Integer(), + "msg": String() +}) +# TODO: 实现 +def del_agent(json_data): + print(json_data) + return {"code": 200, "msg": "ok"} -- cgit v1.2.3 From 97bd83cdc1953a1169ad30cd04b476fa41268ef1 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Tue, 16 Apr 2024 18:17:37 +0800 Subject: sysinfo接口定义完善 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/sysinfo.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/server/apps/sysinfo.py b/server/apps/sysinfo.py index b18cb65..04e5cbf 100644 --- a/server/apps/sysinfo.py +++ b/server/apps/sysinfo.py @@ -37,6 +37,7 @@ class LogOut(Schema): @bp.get("/num") @bp.doc("系统信息获取接口", "返回当前系统状态,包括已部署代理节点数量、已执行任务次数、系统已运行天数以及已探测目标统计") @bp.output({ + "code": Integer(), "agent_num": Integer(), "task_num": Integer(), "workday": Integer(), @@ -56,6 +57,7 @@ def systate(): dot_num = da.count_data("target", {"DOT": True}) # 返回结果 return { + "code": 200, "agent_num": agent_num, "task_num": task_num, "workday": workday, @@ -71,6 +73,7 @@ def systate(): @bp.get("/num/date") @bp.doc("仪表盘柱状图数据获取接口", "返回当前不同探测目标在一周内的数量变化情况") @bp.output({ + "code": Integer(), "date_data": Dict(Date(), Nested(TargetOut())) }) def target_date(): @@ -86,19 +89,26 @@ def target_date(): "dot": da.count_data_by_time("target", time=d, search={"DOT": True}) } day += 1 - return {"date_data": dates} + return {"code": 200, "date_data": dates} # 系统操作日志获取接口 @bp.get("/log") @bp.input({ + "begin": Date(), + "end": Date(), + "level": String(validate=OneOf(["INFO", "WARNING", "ERROR"])), + "user": String(), "per_page": Integer(load_default=10), "page": Integer(load_default=1) }, location="query") @bp.doc("系统操作日志获取接口", "返回系统的操作日志") @bp.output({ - "log_data": List(Nested(LogOut())) + "code": Integer(), + "log_data": List(Nested(LogOut())), + "total": Integer() }) +# TODO:完善 def sys_log(query_data): # 每页显示多少项 per_page = query_data["per_page"] -- cgit v1.2.3 From aa5a97873cafb15c105342bae14c5498d48d7e4e Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Tue, 16 Apr 2024 18:55:05 +0800 Subject: 接口定义同步并完善 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/sysmange.py | 12 ++++++-- server/apps/target.py | 47 +++++++++++++++++++++++++++---- server/apps/task.py | 73 +++++++++++++++++++++++++++++++++---------------- 3 files changed, 101 insertions(+), 31 deletions(-) diff --git a/server/apps/sysmange.py b/server/apps/sysmange.py index 9f391c7..4143d5c 100644 --- a/server/apps/sysmange.py +++ b/server/apps/sysmange.py @@ -23,7 +23,9 @@ class User(Schema): "page": Integer(load_default=1) }, location='query') @bp.output({ - "data": List(Nested(User())) + "code": Integer(), + "data": List(Nested(User())), + "total": Integer() }) def list_user(query_data): # TODO：具体实现 @@ -41,7 +43,7 @@ def list_user(query_data): "created_by": "管理员", "group": "Admin" }) - return {"data": user_list} + return {"code": 200, "data": user_list} @bp.post("/") @@ -52,7 +54,11 @@ def list_user(query_data): "pwd": String(required=True), "cur_user": String(load_default="Root") }) +@bp.output({ + "code": Integer(), + "msg": String() +}) def create_user(json_data): # TODO:具体实现 print(json_data) - return {"msg": "ok"} + return {"code": 200, "msg": "ok"} diff --git a/server/apps/target.py b/server/apps/target.py index 98a5294..0a47599 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -56,6 +56,7 @@ class Delay(Schema): class DelayOut(Schema): + code = Integer() delay_data = List(Nested(Delay())) @@ -77,7 +78,7 @@ def get_pernode_delay(query_data, type): threads.append(mythread) for t in threads: t.join() - return {'delay_data': ans} + return {"code": 200, 'delay_data': ans} threadLock = threading.Lock() @@ -175,6 +176,7 @@ from dns import resolver 'qtype': String(required=True, validate=OneOf(['A', 'AAAA', "CNAME", "NS"])) }, location='query') @bp.output({ + "code": Integer(), "ans": List(Dict(String(validate=ContainsOnly(["rrset"])), String())) }) def record(query_data): @@ -209,18 +211,23 @@ def record(query_data): if myAnswers.rrset is not None: for r in myAnswers.rrset: ans.append({"rrset": str(r)}) - return {'ans': ans} + return {"code": 200, 'ans': ans} @bp.get("/") -@bp.doc("目标信息获取接口", "返回目标信息，已测试") +@bp.doc("(表格)目标信息获取接口", "返回目标信息") @bp.input({ + "protocol": String(validate=OneOf(['IPv6', 'DNSSEC', "DoH", "DoT"])), + "cou": String(), + "isp": String(), "per_page": Integer(load_default=10), "page": Integer(load_default=1), "ip": IP(load_default=None) }, location='query') @bp.output({ - "data": List(Nested(Target())) + "code": Integer(), + "data": List(Nested(Target())), + "total": Integer() }) def target_info(query_data): per_page = query_data["per_page"] @@ -253,4 +260,34 @@ def target_info(query_data): # 原数据的值赋给经过映射之后的键 target[target_map[k]] = r[k] target_list.append(target) - return {"data": target_list} + # TODO:total + return {"code": 200, "data": target_list} + + +@bp.get("/filter") +@bp.doc("目标可筛选信息获取接口") +@bp.output({ + "code": Integer(), + "proto": List(String()), + "isp": List(String()), + "cou": List(String()) +}) +# TODO：实现 +def filter_info(): + proto = ["IPv6", "DNSSEC", "DoH", "DoT"] + isp = ["google", "cloudflare", "阿里云", "DNSPod", "quad9"] + cou = ["美国", "中国", "日本", "澳大利亚", "新加坡"] + return {"code": 200, "proto": proto, "isp": isp, "cou": cou} + + +@bp.get("/map") +@bp.doc("地图信息获取接口", hide=True) +@bp.output({ + "code": Integer(), + "proto": List(String()), + "isp": List(String()), + "cou": List(String()) +}) +def map_info(query_data): + # TODO:实现 + return {"code": 200} diff --git a/server/apps/task.py b/server/apps/task.py index b5edae1..c3d36ce 100644 --- a/server/apps/task.py +++ b/server/apps/task.py @@ -45,15 +45,17 @@ class TaskState(Schema): policy_name = String() # 策略参数 policy_param = String() - # 执行输出 - policy_output = List(Nested(TaskLog())) + # 执行策略编号 + policy_id = String() # 效果评估 policy_status = String() # 创建任务接口 @bp.post("/create") -@bp.doc("任务创建接口", "字段含义详见task.py文件注释") +@bp.doc("任务创建接口", "部分字段值的映射关系：
" + + "policy 期望策略，可选参数范围及对应含义为：，auto-自动，ddos-拒绝服务，sjqp-数据欺骗
" + + "scan 状态感知方式，可选参数范围及对应含义为：auto-自动，icmp-icmp/v6时延，tcp-tcp时延，dns-dns时延，record-记录正确性验证") @bp.input({ # 任务名称 "name": String(), @@ -68,7 +70,7 @@ class TaskState(Schema): # 期望策略 "policy": String(validate=OneOf(["auto", "ddos", "sjqp"])), # 状态感知方式 - "scan": String(validate=OneOf(["auto", "delay", "record"])), + "scan": String(validate=OneOf(["auto", "icmp", "dns", "tcp", "record"])), # 策略切换时限 "policy_time": Integer(), # 任务执行时限 @@ -76,10 +78,14 @@ class TaskState(Schema): # 运行配置 "run_flag": String(validate=OneOf(["now", "man"])) }) +@bp.output({ + "code": Integer(), + "msg": String() +}) # TODO：创建任务接口具体实现 def make_task(json_data): print(json_data) - return {"msg": "ok"} + return {"code": 200, "msg": "ok"} opsmap = {"start": "开始", "stop": "暂停", "cancel": "停止"} @@ -92,10 +98,14 @@ opsmap = {"start": "开始", "stop": "暂停", "cancel": "停止"} "taskid": String(required=True), "ops": String(required=True, validate=OneOf(["start", "stop", "cancel"])) }) +@bp.output({ + "code": Integer(), + "msg": String(), +}) # TODO：操作任务开始停止控制接口具体实现 def ops_task(json_data): ops = opsmap[json_data["ops"]] - return {"msg": "任务" + json_data["taskid"] + "已" + ops} + return {"code": 200, "msg": "任务" + json_data["taskid"] + "已" + ops} # 查询任务列表接口 @@ -106,7 +116,9 @@ def ops_task(json_data): "per_page": Integer(load_default=10) }, location="query") @bp.output({ - "data": List(Nested(Task())) + "code": Integer(), + "data": List(Nested(Task())), + "total": Integer() }) # TODO：查询任务状态接口具体实现 def tasks_state(query_data): @@ -123,7 +135,7 @@ def tasks_state(query_data): "create_time": fake.date_time_between(start_date="-1y"), "status": random.choice(["working", "stop", "finish"]) }) - return {"data": task_list} + return {"code": 200, "data": task_list} # 任务详情接口 @@ -133,6 +145,7 @@ def tasks_state(query_data): "taskid": String(required=True), }, location="query") @bp.output({ + "code": Integer(), "data": List(Nested(TaskState())) }) # TODO：任务详情接口具体实现 @@ -146,13 +159,7 @@ def task_info(query_data): "start_time": fake.date_time_between(start_date="-1y"), "policy_name": random.choice(["IPv6", "DNSSEC", "DoT", "DoH"]) + " " + random.choice(["DDoS", "数据欺骗"]), "policy_param": random.choice(["攻击速率: 1000pps", "目标域名: www.google.com | 目标记录: NS attack.com"]), - "policy_output": [{ - "time": fake.date_time_between(start_date="-1y"), - "ip": "192.168.1.1", - "targetip": "2406:1234:1234:1234:1234:1234:1234:1234", - "level": random.choice(["INFO", "WARNING", "ERROR"]), - "info": fake.text(max_nb_chars=20, ext_word_list=None) - } for _ in range(20)], + "policy_id": str(fake.random.randint(1, 10000)), "policy_status": "无效；原因为：超时未成功" }) # 当前正在执行的策略 @@ -160,13 +167,33 @@ def task_info(query_data): "start_time": datetime.datetime.now(), "policy_name": random.choice(["IPv6", "DNSSEC", "DoT", "DoH"]) + " " + random.choice(["DDoS", "数据欺骗"]), "policy_param": random.choice(["攻击速率: 1000pps", "目标域名: www.google.com | 目标记录: NS attack.com"]), - "policy_output": [{ - "time": fake.date_time_between(start_date="-1y"), - "ip": "192.168.1.1", - "targetip": "2406:1234:1234:1234:1234:1234:1234:1234", - "level": random.choice(["INFO", "WARNING", "ERROR"]), - "info": fake.text(max_nb_chars=20, ext_word_list=None) - } for _ in range(20)], + "policy_id": str(fake.random.randint(1, 10000)), "policy_status": "评估中" }) - return {"data": task_state_list} + return {"code": 200, "data": task_state_list} + + +@bp.get("/tp") +@bp.doc("任务策略执行日志获取接口") +@bp.input({ + "id": String(required=True), + "per_page": Integer(load_default=10), + "page": Integer(load_default=1) +}, location="query") +@bp.output({ + "code": Integer(), + "data": List(Nested(TaskLog())), + "total": Integer() +}) +def taskpolicy_log(query_data): + per_page = query_data["per_page"] + page = query_data["page"] + policy_output = [{ + "time": fake.date_time_between(start_date="-1y"), + "ip": "192.168.1.1", + "targetip": "2406:1234:1234:1234:1234:1234:1234:1234", + "level": random.choice(["INFO", "WARNING", "ERROR"]), + "info": fake.text(max_nb_chars=20, ext_word_list=None) + } for _ in range(per_page)] + + return {"code": 200, "data": policy_output, "total": 10 * per_page} -- cgit v1.2.3 From fdf12ec14b27e4a15ea198a66ac8805fadd41c9a Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 18 Apr 2024 11:16:58 +0800 Subject: 接口实现完善，准备将底层数据库访问工具替换为sqlalchemy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app.py | 8 ++++---- server/apps/agentcomm.py | 51 ++++++++++++++++++++++++++++++------------------ server/apps/model.py | 27 +++++++++++++++++++++++++ server/apps/util.py | 45 ++++++++++++++++++++---------------------- server/requirements.txt | 6 ++++-- 5 files changed, 88 insertions(+), 49 deletions(-) create mode 100644 server/apps/model.py diff --git a/agent/app.py b/agent/app.py index 15d0773..6a26c8c 100644 --- a/agent/app.py +++ b/agent/app.py @@ -19,7 +19,7 @@ app.register_blueprint(scriptbp) config = {} # 代理类型参数映射表 atype_map = { - "stgj": "渗透攻击", + "gjst": "攻击渗透", "csgz": "参数感知", "ztgz": "状态感知" } @@ -67,7 +67,7 @@ def nodeinfo(): # 注册代理 -def registernode(proto="http", port=2525, atype="stgj", server="127.0.0.1:8888"): +def registernode(proto="http", port=2525, atype="gjst", server="127.0.0.1:8888"): info = nodeinfo() info["port"] = port info["atype"] = atype_map[atype] @@ -92,8 +92,8 @@ if __name__ == '__main__': # 命令行参数设置 parser = argparse.ArgumentParser() parser.add_argument("-p", "--port", type=int, default=2525, help="代理的开放通信端口") - parser.add_argument("-t", "--atype", type=str, default="stgj", - help="代理的工作类型 {stgj(渗透攻击) / csgz(参数感知) / ztgz(状态感知)}") + parser.add_argument("-t", "--atype", type=str, default="gjst", + help="代理的工作类型 {gjst(攻击渗透) / csgz(参数感知) / ztgz(状态感知)}") parser.add_argument("-s", "--server", type=str, default="127.0.0.1:8888", help="主控端访问地址+端口号") # 解析参数 diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 4958d57..037deb4 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -12,24 +12,17 @@ from .util import da bp = APIBlueprint("代理管理接口集合", __name__, url_prefix="/agent") # 代理类型 -agent_type = ["渗透攻击", "状态感知", "参数感知"] +agent_type = ["攻击渗透", "状态感知", "参数感知"] +# 代理选项参数和代理类型的映射关系 +agent_key_map = {"gjst": agent_type[0], "ztgz": agent_type[1], "csgz": agent_type[2]} -class agent(Schema): - id = String() - ipaddr = List(String()) - atype = String(validate=OneOf(agent_type)) - status = Boolean() - idle = Boolean() - port = Integer() - sys = String() - cpu_num = Integer() - mem = String() - start_time = DateTime() - +# 状态选项参数和值的映射关系 +status_map = {1: True, 0: False} +idle_map = {1: True, 0: False} # 数据库列与返回值的键对应关系 -agent_map = { +agent_response_map = { "AGENT_ID": "id", "IPADDR": "ipaddr", "START_TIME": "start_time", @@ -45,6 +38,19 @@ agent_map = { } +class agent(Schema): + id = String() + ipaddr = List(String()) + atype = String(validate=OneOf(agent_type)) + status = Boolean() + idle = Boolean() + port = Integer() + sys = String() + cpu_num = Integer() + mem = String() + start_time = DateTime() + + # 代理注册接口 @bp.post("/register") @bp.doc("代理注册接口", "返回分配给代理的编号值", hide=True) @@ -127,9 +133,9 @@ def task_ret(json_data): + "status:当前状态，可选参数范围及对应含义为：2-全部(默认)，1-在线，0-离线
" + "idle:是否空闲，可选参数范围及对应含义为：2-全部(默认)，1-空闲，0-执行中
") @bp.input({ - "atype": String(load_default="all"), - "status": Integer(validate=OneOf([0, 1, 2])), - "idle": Integer(validate=OneOf([0, 1, 2])), + "atype": String(load_default="all", validate=OneOf(["all", "gjst", "ztgz", "csgz"])), + "status": Integer(load_default=2, validate=OneOf([0, 1, 2])), + "idle": Integer(load_default=2, validate=OneOf([0, 1, 2])), "page": Integer(load_default=1), "per_page": Integer(load_default=10) }, location="query") @@ -142,12 +148,19 @@ def task_ret(json_data): def agent_info(query_data): per_page = query_data["per_page"] page = query_data["page"] + agent_type = query_data["atype"] + status = query_data["status"] + idle = query_data["idle"] + + # 代理信息列表 agent_list = [] - res = da.get_data(data_type="agent", offset=(page - 1) * per_page, limit=per_page) + res = da.get_data(data_type="agent", search={"atype": agent_type, "status": status, "idle": idle}, + offset=(page - 1) * per_page, limit=per_page) + res_count = da.count_data(data_type="agent", ) for r in res: agent = {} for key, value in r.items(): - agent[agent_map[key]] = value if key != "IPADDR" else str(value).split("|") + agent[agent_response_map[key]] = value if key != "IPADDR" else str(value).split("|") agent_list.append(agent) return {"code": 200, "agent_data": agent_list, "total": 100} diff --git a/server/apps/model.py b/server/apps/model.py new file mode 100644 index 0000000..276e475 --- /dev/null +++ b/server/apps/model.py @@ -0,0 +1,27 @@ +# 记录参数值和数据库表字段名之间的双向映射 +from bidict import bidict + +from settings import * + +# 参数与数据表映射关系 +tabmapping = bidict({ + "agent": MYSQL_TAB_AGENT, + "task": MYSQL_TAB_TASK, + "user": MYSQL_TAB_USER, + "target": MYSQL_TAB_TARGETDATA, + "syslog": MYSQL_TAB_SYSLOG, }) + +# 参数与数据表中时间字段的映射关系 +timemapping = { + "agent": "START_TIME", + "task": "CREATED_TIME", + "user": "CREATED_TIME", + "target": "UPDATED_TIME", +} + +# +agent_keymapping = bidict({ + "atype": "AGENT_TYPE", + "status": "STATUS", + "idle": "IDLE" +}) diff --git a/server/apps/util.py b/server/apps/util.py index c789f61..399d678 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -8,6 +8,8 @@ from faker.providers import date_time from loguru import logger from pymysql.cursors import DictCursor +import apps.model as model + def debug(message, *args, **kwargs): logger.debug(message, *args, **kwargs) @@ -30,21 +32,6 @@ fake.add_provider(company) # 数据库访问类 class DataHandler: - # 参数与数据表映射关系 - tabmapping = { - "agent": MYSQL_TAB_AGENT, - "task": MYSQL_TAB_TASK, - "user": MYSQL_TAB_USER, - "target": MYSQL_TAB_TARGETDATA, - "syslog": MYSQL_TAB_SYSLOG, } - - # 参数与数据表中时间字段的映射关系 - timemapping = { - "agent": "START_TIME", - "task": "CREATED_TIME", - "user": "CREATED_TIME", - "target": "UPDATED_TIME", - } # 数据库链接及数据库初始化 def __init__(self): @@ -195,24 +182,34 @@ class DataHandler: # 获取信息(代理、任务) # data_type可选范围参照DataHandler.tabmapping的键 # 若需要按条件检索，则以将检索维度与检索值以字典形式传入search - def get_data(self, data_type="agent", offset=0, limit=10, search=None): + def get_data(self, search: dict, data_type="agent", offset=0, limit=10, ): # 参数映射到表名 - tabname = self.tabmapping[data_type] - if search is None: + tabname = model.tabmapping[data_type] + # 比较输入参数和默认值的差异 + differ = set(search.items()).difference({"atype": "all", "status": 2, "idle": 2}.items()) + # 完全一致 + if len(differ) == 0: sql = """SELECT * FROM %s LIMIT %s, %s""" % (tabname, offset, limit) self.cursor.execute(sql) return self.cursor.fetchall() else: - search = dict(search) - key, value = search.popitem() - sql = """SELECT * FROM %s WHERE %s=%s LIMIT %s, %s""" % (tabname, key, value, offset, limit) + l = len(differ) + # 条件字典 + condition = {} + for _ in range(l): + key, val = differ.pop() + tab_key = model.agent_keymapping[key] + condition[tab_key] = str(val) + sql = """SELECT * FROM %s WHERE %s LIMIT %s, %s""" % ( + tabname, "AND".join(["=".join(condition.popitem()) for _ in range(l)]), offset, limit) + print(sql) self.cursor.execute(sql) return self.cursor.fetchall() # 统计符合条件的信息数量 def count_data(self, data_type="agent", search=None): # 参数映射到表名 - tabname = self.tabmapping[data_type] + tabname = model.tabmapping[data_type] if search is None: sql = """SELECT count(*) FROM %s""" % (tabname) self.cursor.execute(sql) @@ -225,8 +222,8 @@ class DataHandler: return dict(self.cursor.fetchall()[0]).popitem()[1] def count_data_by_time(self, data_type="agent", time=None, search=None): - tabname = self.tabmapping[data_type] - timename = self.timemapping[data_type] + tabname = model.tabmapping[data_type] + timename = model.timemapping[data_type] key, value = search.popitem() sql = """SELECT COUNT(*) FROM %s WHERE DATE_FORMAT(%s,'%Y-%m-%d')=%s AND %s=%s""" % (tabname, timename, time, key, value) diff --git a/server/requirements.txt b/server/requirements.txt index 56babab..c239b02 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -2,10 +2,12 @@ requests~=2.31.0 Flask~=3.0.0 APIFlask~=2.1.0 click~=8.1.7 -numpy~=1.26.3 +numpy~=1.25.0 six~=1.16.0 pandas~=2.1.4 faker~=18.9.0 dnspython~=2.6.1 pymysql~=1.1.0 -loguru~=0.5.3 \ No newline at end of file +loguru~=0.5.3 +flask-cors +bidict~=0.23.1 \ No newline at end of file -- cgit v1.2.3 From 39075842eca4562f4a72a42195f4c9dae2d30cc8 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 18 Apr 2024 12:13:58 +0800 Subject: sqlalchemy替换计划作废，原因在于无法返回字典结果 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 完善条件查询语句 --- server/apps/model.py | 6 ++++++ server/apps/util.py | 33 +++++++++++++++++---------------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/server/apps/model.py b/server/apps/model.py index 276e475..25b6464 100644 --- a/server/apps/model.py +++ b/server/apps/model.py @@ -25,3 +25,9 @@ agent_keymapping = bidict({ "status": "STATUS", "idle": "IDLE" }) + +typemapping = { + "atype": "str", + "status": "int", + "idle": "int" +} diff --git a/server/apps/util.py b/server/apps/util.py index 399d678..7d72138 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -9,20 +9,6 @@ from loguru import logger from pymysql.cursors import DictCursor import apps.model as model - - -def debug(message, *args, **kwargs): - logger.debug(message, *args, **kwargs) - - -def info(message, *args, **kwargs): - logger.info(message, *args, **kwargs) - - -def error(message, *args, **kwargs): - logger.error(message, *args, **kwargs) - - from settings import * fake = Faker("zh_CN") @@ -199,9 +185,12 @@ class DataHandler: for _ in range(l): key, val = differ.pop() tab_key = model.agent_keymapping[key] - condition[tab_key] = str(val) + if model.typemapping[key] == "str": + condition[tab_key] = "\"" + val + "\"" + else: + condition[tab_key] = str(val) sql = """SELECT * FROM %s WHERE %s LIMIT %s, %s""" % ( - tabname, "AND".join(["=".join(condition.popitem()) for _ in range(l)]), offset, limit) + tabname, " AND ".join(["=".join(condition.popitem()) for _ in range(l)]), offset, limit) print(sql) self.cursor.execute(sql) return self.cursor.fetchall() @@ -232,3 +221,15 @@ class DataHandler: da = DataHandler() + + +def debug(message, *args, **kwargs): + logger.debug(message, *args, **kwargs) + + +def info(message, *args, **kwargs): + logger.info(message, *args, **kwargs) + + +def error(message, *args, **kwargs): + logger.error(message, *args, **kwargs) -- cgit v1.2.3 From 3c02761ae60e96feca4829ef4f2d73e07b8e8637 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 18 Apr 2024 12:27:41 +0800 Subject: 代理信息获取接口实现完善 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 5 ++--- server/apps/model.py | 5 +++++ server/apps/util.py | 49 +++++++++++++++++++++++++----------------------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 037deb4..43f197b 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -144,7 +144,6 @@ def task_ret(json_data): "agent_data": List(Nested(agent())), "total": Integer() }) -# TODO: code total def agent_info(query_data): per_page = query_data["per_page"] page = query_data["page"] @@ -156,13 +155,13 @@ def agent_info(query_data): agent_list = [] res = da.get_data(data_type="agent", search={"atype": agent_type, "status": status, "idle": idle}, offset=(page - 1) * per_page, limit=per_page) - res_count = da.count_data(data_type="agent", ) + res_count = da.count_data(data_type="agent", search={"atype": agent_type, "status": status, "idle": idle}) for r in res: agent = {} for key, value in r.items(): agent[agent_response_map[key]] = value if key != "IPADDR" else str(value).split("|") agent_list.append(agent) - return {"code": 200, "agent_data": agent_list, "total": 100} + return {"code": 200, "agent_data": agent_list, "total": res_count} # 代理信息存储到数据库 diff --git a/server/apps/model.py b/server/apps/model.py index 25b6464..45ea54d 100644 --- a/server/apps/model.py +++ b/server/apps/model.py @@ -31,3 +31,8 @@ typemapping = { "status": "int", "idle": "int" } + +# 默认参数 +default_data = { + "agent": {"atype": "all", "status": 2, "idle": 2} +} diff --git a/server/apps/util.py b/server/apps/util.py index 7d72138..320ef37 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -168,16 +168,24 @@ class DataHandler: # 获取信息(代理、任务) # data_type可选范围参照DataHandler.tabmapping的键 # 若需要按条件检索，则以将检索维度与检索值以字典形式传入search - def get_data(self, search: dict, data_type="agent", offset=0, limit=10, ): + def get_data(self, search: dict = None, data_type="agent", offset=0, limit=10, count=False): # 参数映射到表名 tabname = model.tabmapping[data_type] # 比较输入参数和默认值的差异 - differ = set(search.items()).difference({"atype": "all", "status": 2, "idle": 2}.items()) + if search == None: + differ = set() + else: + differ = set(search.items()).difference(model.default_data[data_type].items()) # 完全一致 if len(differ) == 0: - sql = """SELECT * FROM %s LIMIT %s, %s""" % (tabname, offset, limit) - self.cursor.execute(sql) - return self.cursor.fetchall() + if not count: + sql = """SELECT * FROM %s LIMIT %s, %s""" % (tabname, offset, limit) + self.cursor.execute(sql) + return self.cursor.fetchall() + else: + sql = """SELECT count(*) FROM %s""" % (tabname) + self.cursor.execute(sql) + return dict(self.cursor.fetchall()[0]).popitem()[1] else: l = len(differ) # 条件字典 @@ -189,26 +197,21 @@ class DataHandler: condition[tab_key] = "\"" + val + "\"" else: condition[tab_key] = str(val) - sql = """SELECT * FROM %s WHERE %s LIMIT %s, %s""" % ( - tabname, " AND ".join(["=".join(condition.popitem()) for _ in range(l)]), offset, limit) - print(sql) - self.cursor.execute(sql) - return self.cursor.fetchall() + if not count: + sql = """SELECT * FROM %s WHERE %s LIMIT %s, %s""" % ( + tabname, " AND ".join(["=".join(condition.popitem()) for _ in range(l)]), offset, limit) + print(sql) + self.cursor.execute(sql) + return self.cursor.fetchall() + else: + sql = """SELECT count(*) FROM %s WHERE %s LIMIT %s, %s""" % ( + tabname, " AND ".join(["=".join(condition.popitem()) for _ in range(l)]), offset, limit) + self.cursor.execute(sql) + return dict(self.cursor.fetchall()[0]).popitem()[1] # 统计符合条件的信息数量 - def count_data(self, data_type="agent", search=None): - # 参数映射到表名 - tabname = model.tabmapping[data_type] - if search is None: - sql = """SELECT count(*) FROM %s""" % (tabname) - self.cursor.execute(sql) - return dict(self.cursor.fetchall()[0]).popitem()[1] - else: - search = dict(search) - key, value = search.popitem() - sql = """SELECT count(*) FROM %s WHERE %s=%s""" % (tabname, key, value) - self.cursor.execute(sql) - return dict(self.cursor.fetchall()[0]).popitem()[1] + def count_data(self, search: dict = None, data_type="agent"): + return self.get_data(search=search, data_type=data_type, count=True) def count_data_by_time(self, data_type="agent", time=None, search=None): tabname = model.tabmapping[data_type] -- cgit v1.2.3 From b862fe9d30bf179cefaddaf25f8ccfc8b50b36d2 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 18 Apr 2024 17:18:12 +0800 Subject: 代理注册接口问题修复 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app.py | 4 +-- server/apps/agentcomm.py | 82 +++++++++++++++++++++++++++--------------------- server/apps/util.py | 11 +++++++ 3 files changed, 60 insertions(+), 37 deletions(-) diff --git a/agent/app.py b/agent/app.py index 6a26c8c..ee88577 100644 --- a/agent/app.py +++ b/agent/app.py @@ -100,8 +100,8 @@ if __name__ == '__main__': args = parser.parse_args() # 注册代理，并获取主控分配的代理编号 - id = registernode(port=args.port, server=args.server, atype=args.type) - config = {"id": id, "port": args.port, "atype": args.type, "server": args.server} + id = registernode(port=args.port, server=args.server, atype=args.atype) + config = {"id": id, "port": args.port, "atype": args.atype, "server": args.server} # 配置写入yaml文件存储 with open('config.yaml', 'w') as f: diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 43f197b..d8611a9 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -7,7 +7,7 @@ from apiflask.validators import OneOf from flask import request from settings import * -from .util import da +from .util import da, error, string_to_mysql bp = APIBlueprint("代理管理接口集合", __name__, url_prefix="/agent") @@ -78,18 +78,22 @@ def register_agent(): id = "".join(random.sample('1234567890zyxwvutsrqponmlkjihgfedcba', 8)) # 代理所有参数 param = {"id": id, - "ipaddr": (request.remote_addr).join(("|" + i) for i in data["v6addr"]), - "start_time": str(datetime.datetime.now()), + "ipaddr": "|".join([request.remote_addr] + [i for i in data["v6addr"]]), + # "start_time",在数据库中实现 "lat": data["lat"], "lng": data["lng"], "cpunum": data["cpu_num"], - "mem": data["ram_size"] + "GB" + "(" + {1 - data["ram_per"]} + "%" + "可用)", + "mem": str(data["ram_size"]) + "GB" + "(" + str(1 - data["ram_per"]) + "%" + "可用)", "sys": data["sys"], "status": True, "idle": True, "port": data["port"], "atype": data["atype"], } + # 对字符串值进行处理 + param = string_to_mysql(param) + param["tab"] = MYSQL_TAB_AGENT + print(param) err = insert_agent(param) if not err: return {"id": id} @@ -164,37 +168,6 @@ def agent_info(query_data): return {"code": 200, "agent_data": agent_list, "total": res_count} -# 代理信息存储到数据库 -def insert_agent(param: dict): - sql = """REPLACE INTO %s(AGENT_ID,IPADDR, - CREATED_TIME, - LAT, - LNG , - AGENT_TYPE, - SYS , - PORT , - CPU_NUM , - STATUS , - RAM_SIZE , - WORK_STATUS) VALUES( - %(id)s, - %(ipaddr)s, - %(create_time)s, - %(lat)s, - %(lng)s, - %(atype)s, - %(sys)s, - %(port)s, - %(cpunum)s, - %(status)s, - %(mem)s, - %(work_status)s) - )""" % param - da.cursor.execute(sql) - da.conn.commit() - return None - - @bp.doc("代理删除接口", "输入参数说明：
" + "id: 代理编号") @bp.post("/del") @@ -208,4 +181,43 @@ def insert_agent(param: dict): # TODO: 实现 def del_agent(json_data): print(json_data) + sql = """DELETE FROM %s WHERE id = %s + """ % (MYSQL_TAB_AGENT, json_data["id"]) + try: + da.cursor.execute(sql) + da.conn.commit() + except Exception as e: + error(message=str(e)) return {"code": 200, "msg": "ok"} + + +# 代理信息存储到数据库 +def insert_agent(param: dict): + sql = """REPLACE INTO %(tab)s( + AGENT_ID, + IPADDR, + LAT, + LNG , + AGENT_TYPE, + SYS , + PORT , + CPU_NUM , + STATUS , + MEM, + IDLE) + VALUES( + %(id)s, + %(ipaddr)s, + %(lat)s, + %(lng)s, + %(atype)s, + %(sys)s, + %(port)s, + %(cpunum)s, + %(status)s, + %(mem)s, + %(idle)s);""" % param + print(sql) + da.cursor.execute(sql) + da.conn.commit() + return None diff --git a/server/apps/util.py b/server/apps/util.py index 320ef37..39986cb 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -226,6 +226,17 @@ class DataHandler: da = DataHandler() +# 将插入mysql中的字符串处理为满足mysql形式的单引号字符串，输入为字典形式 +def string_to_mysql(data: dict): + res = {} + for key, val in data.items(): + if type(val) == type("text"): + res[key] = "\'" + val + "\'" + else: + res[key] = val + return res + + def debug(message, *args, **kwargs): logger.debug(message, *args, **kwargs) -- cgit v1.2.3 From 00002e3d2ae25db396e33899c6baa9e6c77ff865 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 18 Apr 2024 17:19:39 +0800 Subject: 测试代码删除 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index d8611a9..a3876f7 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -92,8 +92,10 @@ def register_agent(): } # 对字符串值进行处理 param = string_to_mysql(param) + # 数据库表名 param["tab"] = MYSQL_TAB_AGENT - print(param) + + # 插入记录 err = insert_agent(param) if not err: return {"id": id} @@ -217,7 +219,6 @@ def insert_agent(param: dict): %(status)s, %(mem)s, %(idle)s);""" % param - print(sql) da.cursor.execute(sql) da.conn.commit() return None -- cgit v1.2.3 From f2a3d11fb8606733f46d5e88ed86957433db3eea Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 18 Apr 2024 17:22:05 +0800 Subject: 代理删除接口完善 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index a3876f7..51af7f3 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -183,13 +183,14 @@ def agent_info(query_data): # TODO: 实现 def del_agent(json_data): print(json_data) - sql = """DELETE FROM %s WHERE id = %s + sql = """DELETE FROM %s WHERE AGENT_ID = '%s' """ % (MYSQL_TAB_AGENT, json_data["id"]) try: da.cursor.execute(sql) da.conn.commit() except Exception as e: error(message=str(e)) + return {"code": 500, "msg": str(e)} return {"code": 200, "msg": "ok"} @@ -219,6 +220,9 @@ def insert_agent(param: dict): %(status)s, %(mem)s, %(idle)s);""" % param - da.cursor.execute(sql) - da.conn.commit() + try: + da.cursor.execute(sql) + da.conn.commit() + except Exception as e: + error(message=str(e)) return None -- cgit v1.2.3 From b5b29877caf54ad5f2be76d5eb18286020ee5d60 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 18 Apr 2024 17:24:23 +0800 Subject: 代理接口完善 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 51af7f3..48d0fdd 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -99,11 +99,13 @@ def register_agent(): err = insert_agent(param) if not err: return {"id": id} + else: + return {"code": 500, "err": err} # 代理任务下发 +# TODO:具体实现 def deliver_task(agent_id, policy, policy_param): - # TODO:具体实现 # 查询agent_id对应代理的ip和端口信息 # 将policy和policy_param作为参数传递到对应接口 pass @@ -180,7 +182,6 @@ def agent_info(query_data): "code": Integer(), "msg": String() }) -# TODO: 实现 def del_agent(json_data): print(json_data) sql = """DELETE FROM %s WHERE AGENT_ID = '%s' @@ -225,4 +226,5 @@ def insert_agent(param: dict): da.conn.commit() except Exception as e: error(message=str(e)) + return e return None -- cgit v1.2.3 From 682b274f8815ab431d8117be261eae43a223b704 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Thu, 18 Apr 2024 22:12:29 +0800 Subject: 代理接口问题修复 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/agentcomm.py | 26 ++++++++++++++++++++------ server/apps/util.py | 4 ++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/server/apps/agentcomm.py b/server/apps/agentcomm.py index 48d0fdd..4a224ac 100644 --- a/server/apps/agentcomm.py +++ b/server/apps/agentcomm.py @@ -2,7 +2,7 @@ import random from apiflask import APIBlueprint, Schema -from apiflask.fields import String, Integer, List, Nested, Boolean, DateTime +from apiflask.fields import String, Integer, List, Nested, Boolean, DateTime, Float from apiflask.validators import OneOf from flask import request @@ -53,10 +53,23 @@ class agent(Schema): # 代理注册接口 @bp.post("/register") -@bp.doc("代理注册接口", "返回分配给代理的编号值", hide=True) +@bp.doc("代理注册接口", "返回分配给代理的编号值，用于代理和主控端通信注册，前端界面无需调用") +@bp.input({ + "atype": String(), + 'v6addr': List(String()), + 'lat': String(), + 'lng': String(), + 'cpu_num': Integer(), + 'ram_size': Integer(), + 'ram_per': Float(), + 'sys': String(), + "port": Integer() +}, example={'atype': "gjst", 'v6addr': ["2001::1"], 'lat': "-1.111", 'lng': "1.1111", 'cpu_num': 4, + 'ram_size': 16, 'ram_per': 0.55, + 'sys': "linux", "port": 2525}) # 参数说明 # 代理类型 -# "type": String(required=True, validate=OneOf(["stgj", "mbgz", "ztgz"])), +# "type": String(required=True, validate=OneOf(["gjst", "mbgz", "ztgz"])), # 代理的通信端口 # "port": Integer(required=True), # # 代理所在的操作系统 @@ -73,8 +86,8 @@ class agent(Schema): # "ram_size": Integer(), # # 已用内存比例 # "ram_per": Float() -def register_agent(): - data = dict(request.json) +def register_agent(json_data): + data = dict(json_data) id = "".join(random.sample('1234567890zyxwvutsrqponmlkjihgfedcba', 8)) # 代理所有参数 param = {"id": id, @@ -83,7 +96,8 @@ def register_agent(): "lat": data["lat"], "lng": data["lng"], "cpunum": data["cpu_num"], - "mem": str(data["ram_size"]) + "GB" + "(" + str(1 - data["ram_per"]) + "%" + "可用)", + "mem": str(data["ram_size"]) + "GB" + "(" + str( + '{:.2f}'.format(100 * (1 - float(data["ram_per"])))) + "%" + "可用)", "sys": data["sys"], "status": True, "idle": True, diff --git a/server/apps/util.py b/server/apps/util.py index 39986cb..96711ec 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -179,7 +179,7 @@ class DataHandler: # 完全一致 if len(differ) == 0: if not count: - sql = """SELECT * FROM %s LIMIT %s, %s""" % (tabname, offset, limit) + sql = """SELECT * FROM %s LIMIT %s, %s""" % (tabname, offset, offset + limit) self.cursor.execute(sql) return self.cursor.fetchall() else: @@ -199,7 +199,7 @@ class DataHandler: condition[tab_key] = str(val) if not count: sql = """SELECT * FROM %s WHERE %s LIMIT %s, %s""" % ( - tabname, " AND ".join(["=".join(condition.popitem()) for _ in range(l)]), offset, limit) + tabname, " AND ".join(["=".join(condition.popitem()) for _ in range(l)]), offset, offset + limit) print(sql) self.cursor.execute(sql) return self.cursor.fetchall() -- cgit v1.2.3 From c9700eabaa42c21367221bbffe01ada52fe7362b Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Sat, 20 Apr 2024 21:51:14 +0800 Subject: 完善目标信息获取接口 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/model.py | 24 +++++++++++++++++------- server/apps/target.py | 23 +++++++++++++++-------- server/apps/task.py | 41 +++++++++++++++++++++++++++-------------- server/apps/util.py | 21 +++++++++++++++++---- 4 files changed, 76 insertions(+), 33 deletions(-) diff --git a/server/apps/model.py b/server/apps/model.py index 45ea54d..45cf61a 100644 --- a/server/apps/model.py +++ b/server/apps/model.py @@ -19,12 +19,21 @@ timemapping = { "target": "UPDATED_TIME", } -# -agent_keymapping = bidict({ - "atype": "AGENT_TYPE", - "status": "STATUS", - "idle": "IDLE" -}) +# 不同接口的参数（或值）和对应数据表字段的映射关系 +keymapping = { + "agent": bidict({ + "atype": "AGENT_TYPE", + "status": "STATUS", + "idle": "IDLE" + }), + "target": bidict({ + "cou": "COU", + "isp": "ISP", + "IPv6": "IPv6", + "DNSSEC": "DNSSEC", + "DoH": "DOH", + "DoT": "DOT", + })} typemapping = { "atype": "str", @@ -34,5 +43,6 @@ typemapping = { # 默认参数 default_data = { - "agent": {"atype": "all", "status": 2, "idle": 2} + "agent": {"atype": "all", "status": 2, "idle": 2}, + "target": {"proto": None, "isp": None, "cou": None} } diff --git a/server/apps/target.py b/server/apps/target.py index 0a47599..05c1263 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -217,9 +217,9 @@ def record(query_data): @bp.get("/") @bp.doc("(表格)目标信息获取接口", "返回目标信息") @bp.input({ - "protocol": String(validate=OneOf(['IPv6', 'DNSSEC', "DoH", "DoT"])), - "cou": String(), - "isp": String(), + "protocol": String(load_default=None, validate=OneOf(['IPv6', 'DNSSEC', "DoH", "DoT"])), + "cou": String(load_default=None), + "isp": String(load_default=None), "per_page": Integer(load_default=10), "page": Integer(load_default=1), "ip": IP(load_default=None) @@ -232,16 +232,24 @@ def record(query_data): def target_info(query_data): per_page = query_data["per_page"] page = query_data["page"] + proto = query_data["protocol"] + cou = query_data["cou"] + isp = query_data["isp"] + ip = query_data["ip"] + + # 目标信息列表 target_list = [] - if query_data["ip"] is None: + if ip is None: # 普通检索 - res = da.get_data(data_type="target", offset=(page - 1) * per_page, limit=per_page) + res = da.get_data(data_type="target", search={"proto": proto, "cou": cou, "isp": isp}, + offset=(page - 1) * per_page, limit=per_page) + res_count = da.count_data(data_type="target", search={"proto": proto, "cou": cou, "isp": isp}) else: # 查询目标 res = da.get_data(data_type="target", offset=(page - 1) * per_page, limit=per_page, search={"ip": query_data["ip"]}) - + res_count = 1 # 结果转换 for r in res: target = {} @@ -260,8 +268,7 @@ def target_info(query_data): # 原数据的值赋给经过映射之后的键 target[target_map[k]] = r[k] target_list.append(target) - # TODO:total - return {"code": 200, "data": target_list} + return {"code": 200, "data": target_list, "total": res_count} @bp.get("/filter") diff --git a/server/apps/task.py b/server/apps/task.py index c3d36ce..751a4a1 100644 --- a/server/apps/task.py +++ b/server/apps/task.py @@ -6,10 +6,23 @@ from apiflask import APIBlueprint, Schema from apiflask.fields import String, Integer, IP, DateTime, List, Nested from apiflask.validators import OneOf -from .util import fake +from .util import fake, da bp = APIBlueprint("任务管理接口集合", __name__, url_prefix="/task") +# 数据库列与返回值的键对应关系 +task_response_map = { + "TASK_ID": "id", + "TARGET_IP": "target", + "TASK_NAME": "name", + "AGENT_ID": "agent", + "TARGET_DOMAIN": "target_domain", + "TARGET_RR": "target_rr", + "POLICY": "policy", + "CREATE_TIME": "create_time", + "STATUS": "status", +} + class Task(Schema): id = Integer() @@ -122,20 +135,20 @@ def ops_task(json_data): }) # TODO：查询任务状态接口具体实现 def tasks_state(query_data): + per_page = query_data["per_page"] + page = query_data["page"] + + # 任务列表 task_list = [] - for _ in range(query_data["per_page"]): - task_list.append({ - "id": fake.random.randint(1, 10000), - "target": fake.ipv4(), - "name": "示例任务", - "agent": fake.random.randint(1, 10000), - "target_domain": fake.domain_name(), - "target_rr": "NS attack.com", - "policy": random.choice(["auto", "ddos", "sjqp"]), - "create_time": fake.date_time_between(start_date="-1y"), - "status": random.choice(["working", "stop", "finish"]) - }) - return {"code": 200, "data": task_list} + res = da.get_data(data_type="task", + offset=(page - 1) * per_page, limit=per_page) + res_count = da.count_data(data_type="task") + for r in res: + task = {} + for key, value in r.items(): + task[task_response_map[key]] = value + task_list.append(task) + return {"code": 200, "data": task_list, "total": res_count} # 任务详情接口 diff --git a/server/apps/util.py b/server/apps/util.py index 96711ec..a608782 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -192,11 +192,24 @@ class DataHandler: condition = {} for _ in range(l): key, val = differ.pop() - tab_key = model.agent_keymapping[key] - if model.typemapping[key] == "str": - condition[tab_key] = "\"" + val + "\"" + # target表的协议参数和 ip 单独处理 + if data_type == "target" and key == "proto": + tab_key = model.keymapping[data_type][val] + condition[tab_key] = True + if data_type == "target" and key == "ip": + if "." in val: + condition["ADDRv4"] = "\"" + val + "\"" + elif ":" in val: + condition["ADDRv6"] = "\"" + val + "\"" + else: + error("错误的地址输入： " + str(val)) else: - condition[tab_key] = str(val) + # 参数在数据表中对应的字段名 + tab_key = model.keymapping[data_type][key] + if model.typemapping[key] == "str": + condition[tab_key] = "\"" + val + "\"" + else: + condition[tab_key] = str(val) if not count: sql = """SELECT * FROM %s WHERE %s LIMIT %s, %s""" % ( tabname, " AND ".join(["=".join(condition.popitem()) for _ in range(l)]), offset, offset + limit) -- cgit v1.2.3 From a1be40004d1cdeaf5b8f6096fb569d0415fdcc41 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Mon, 22 Apr 2024 10:15:30 +0800 Subject: 完成目标信息获取接口功能实现 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/model.py | 14 +++++++++++++- server/apps/target.py | 30 ++++++++++++++++++++++-------- server/apps/util.py | 22 +++++----------------- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/server/apps/model.py b/server/apps/model.py index 45cf61a..0748f70 100644 --- a/server/apps/model.py +++ b/server/apps/model.py @@ -33,12 +33,24 @@ keymapping = { "DNSSEC": "DNSSEC", "DoH": "DOH", "DoT": "DOT", + "ADDRv4": "ADDRv4", + "ADDRv6": "ADDRv6" })} +# 所有参数在数据库中对应的数据类型，用于拼接sql语句时特殊处理 typemapping = { "atype": "str", "status": "int", - "idle": "int" + "idle": "int", + "cou": "str", + "isp": "str", + "ip": "str", + "IPv6": "int", + "DNSSEC": "int", + "DoH": "int", + "DoT": "int", + "ADDRv4": "str", + "ADDRv6": "str" } # 默认参数 diff --git a/server/apps/target.py b/server/apps/target.py index 05c1263..6c4e667 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -8,6 +8,7 @@ from apiflask import APIBlueprint, Schema from apiflask.fields import Integer, String, List, Nested, IP, DateTime, Dict from requests.exceptions import Timeout +from settings import * from .util import da, debug bp = APIBlueprint("目标信息及状态接口集合", __name__, url_prefix="/target") @@ -240,15 +241,22 @@ def target_info(query_data): # 目标信息列表 target_list = [] + # 普通检索，默认所有条件为单选 if ip is None: - # 普通检索 - res = da.get_data(data_type="target", search={"proto": proto, "cou": cou, "isp": isp}, - offset=(page - 1) * per_page, limit=per_page) - res_count = da.count_data(data_type="target", search={"proto": proto, "cou": cou, "isp": isp}) + # 无协议筛选 + if proto == None: + res = da.get_data(data_type="target", search={"proto": proto, "cou": cou, "isp": isp}, + offset=(page - 1) * per_page, limit=per_page) + res_count = da.count_data(data_type="target", search={"proto": proto, "cou": cou, "isp": isp}) + # 协议筛选，填入筛选的协议 + else: + res = da.get_data(data_type="target", search={proto: True, "cou": cou, "isp": isp}, + offset=(page - 1) * per_page, limit=per_page) + res_count = da.count_data(data_type="target", search={proto: True, "cou": cou, "isp": isp}) else: - # 查询目标 - res = da.get_data(data_type="target", offset=(page - 1) * per_page, limit=per_page, - search={"ip": query_data["ip"]}) + # 查询目标，根据v4、v6地址分类 + res = da.get_data(data_type="target", + search={"ADDRv4": ip} if "." in str(ip) else {"ADDRv6": ip}) res_count = 1 # 结果转换 for r in res: @@ -282,7 +290,13 @@ def target_info(query_data): # TODO：实现 def filter_info(): proto = ["IPv6", "DNSSEC", "DoH", "DoT"] - isp = ["google", "cloudflare", "阿里云", "DNSPod", "quad9"] + isp_sql = """SELECT DISTINCT ISP from %s """ % MYSQL_TAB_TARGETDATA + # 执行查询 + da.cursor.execute(isp_sql) + isp_data = da.cursor.fetchall() + # TODO：测试 + isp = [i[0] for i in isp_data] + cou = ["美国", "中国", "日本", "澳大利亚", "新加坡"] return {"code": 200, "proto": proto, "isp": isp, "cou": cou} diff --git a/server/apps/util.py b/server/apps/util.py index a608782..7a7932f 100644 --- a/server/apps/util.py +++ b/server/apps/util.py @@ -192,24 +192,12 @@ class DataHandler: condition = {} for _ in range(l): key, val = differ.pop() - # target表的协议参数和 ip 单独处理 - if data_type == "target" and key == "proto": - tab_key = model.keymapping[data_type][val] - condition[tab_key] = True - if data_type == "target" and key == "ip": - if "." in val: - condition["ADDRv4"] = "\"" + val + "\"" - elif ":" in val: - condition["ADDRv6"] = "\"" + val + "\"" - else: - error("错误的地址输入： " + str(val)) + # 参数在数据表中对应的字段名 + tab_key = model.keymapping[data_type][key] + if model.typemapping[key] == "str": + condition[tab_key] = "\"".join(["", str(val), ""]) else: - # 参数在数据表中对应的字段名 - tab_key = model.keymapping[data_type][key] - if model.typemapping[key] == "str": - condition[tab_key] = "\"" + val + "\"" - else: - condition[tab_key] = str(val) + condition[tab_key] = str(val) if not count: sql = """SELECT * FROM %s WHERE %s LIMIT %s, %s""" % ( tabname, " AND ".join(["=".join(condition.popitem()) for _ in range(l)]), offset, offset + limit) -- cgit v1.2.3 From db8be15a5bd26f992f282f8fb8598ed09ae01070 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Mon, 22 Apr 2024 10:22:05 +0800 Subject: 完成目标可筛选信息获取接口功能实现 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/target.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/apps/target.py b/server/apps/target.py index 6c4e667..98e1cb1 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -287,17 +287,21 @@ def target_info(query_data): "isp": List(String()), "cou": List(String()) }) -# TODO：实现 def filter_info(): proto = ["IPv6", "DNSSEC", "DoH", "DoT"] + # 查询所有的isp isp_sql = """SELECT DISTINCT ISP from %s """ % MYSQL_TAB_TARGETDATA # 执行查询 da.cursor.execute(isp_sql) isp_data = da.cursor.fetchall() - # TODO：测试 - isp = [i[0] for i in isp_data] + isp = [i.popitem()[1] for i in isp_data] - cou = ["美国", "中国", "日本", "澳大利亚", "新加坡"] + # 查询所有的国家 + cou_sql = """SELECT DISTINCT COU from %s """ % MYSQL_TAB_TARGETDATA + # 执行查询 + da.cursor.execute(cou_sql) + cou_data = da.cursor.fetchall() + cou = [i.popitem()[1] for i in cou_data] return {"code": 200, "proto": proto, "isp": isp, "cou": cou} -- cgit v1.2.3 From 3d3b8e07a34390f3941c17e5bdf92dfbadbdaee0 Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Mon, 22 Apr 2024 17:09:00 +0800 Subject: 完成地图信息获取接口功能实现 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/apps/target.py | 94 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 6 deletions(-) diff --git a/server/apps/target.py b/server/apps/target.py index 98e1cb1..5a1875e 100644 --- a/server/apps/target.py +++ b/server/apps/target.py @@ -61,6 +61,16 @@ class DelayOut(Schema): delay_data = List(Nested(Delay())) +# 地图统计信息返回结构体 +class CouInfo(Schema): + # 国家名称 + name = String() + # 不同协议的解析器详细数量说明 + title = String() + # 总数量 + value = Integer() + + @bp.get("/delay/") @bp.doc("获取每个节点的时延数据", "type参数为{icmp,dns,tcp}中的一个", hide=True) @bp.input({"ip": IP(required=False)}, location="query") @@ -306,13 +316,85 @@ def filter_info(): @bp.get("/map") -@bp.doc("地图信息获取接口", hide=True) +@bp.doc("地图信息获取接口") +@bp.input({ + "protocol": String(load_default=None, validate=OneOf(['IPv6', 'DNSSEC', "DoH", "DoT"])), + "cou": String(load_default=None), + "isp": String(load_default=None), + "ip": IP(load_default=None) +}, location="query") @bp.output({ "code": Integer(), - "proto": List(String()), - "isp": List(String()), - "cou": List(String()) + "dataObject": Dict(String(load_default="earthAddTitle"), List(Nested(CouInfo()))), }) def map_info(query_data): - # TODO:实现 - return {"code": 200} + ip = query_data["ip"] + proto = query_data["protocol"] + cou = query_data["cou"] + isp = query_data["isp"] + + if ip is None: + cou_list = [] + # 比较和默认参数值不同的参数 + differ = set({"proto": proto, "cou": cou, "isp": isp}.items()).difference( + {"proto": None, "cou": None, "isp": None}.items()) + if differ == set(): + # 在数据表中进行分组查询,无额外筛选 + sql = """ + SELECT T.COU,sum(T.c) as R,sum(T.dnssec) as DNSSEC,sum(T.v6) as IPv6,sum(T.dh) as DoH,sum(T.dt) as DoT + FROM + (SELECT COU,count(*) as c,count(DNSSEC=1 or null) as dnssec,count(IPv6=1 or null) as v6,count(DOH=1 or null) as dh,count(DOT=1 or null) as dt + FROM %s + GROUP BY COU,DNSSEC,IPv6,DOH,DOT) as T GROUP BY T.COU;""" % (MYSQL_TAB_TARGETDATA) + else: + l = len(differ) + # 条件字典 + condition = {} + for _ in range(l): + key, val = differ.pop() + # 协议参数在数据表中对应的字段名 + if key == "proto": + condition[{'IPv6': "IPv6", "DNSSEC": "DNSSEC", "DoH": "DOH", "DoT": "DOT"}[val]] = str(True) + else: + # 国家和isp属性的键只需调整字母为大写 + key = key.upper() + condition[key] = "\"".join(["", str(val), ""]) + # 在数据表中进行分组查询,无额外筛选 + sql = """ + SELECT T.COU,sum(T.c) as R,sum(T.dnssec) as DNSSEC,sum(T.v6) as IPv6,sum(T.dh) as DoH,sum(T.dt) as DoT + FROM + (SELECT COU,count(*) as c,count(DNSSEC=1 or null) as dnssec,count(IPv6=1 or null) as v6,count(DOH=1 or null) as dh,count(DOT=1 or null) as dt + FROM %s + WHERE %s + GROUP BY COU,DNSSEC,IPv6,DOH,DOT) as T GROUP BY T.COU;""" % ( + MYSQL_TAB_TARGETDATA, " AND ".join(["=".join(condition.popitem()) for _ in range(l)])) + + # 执行查询 + da.cursor.execute(sql) + data = da.cursor.fetchall() + for d in data: + # 单一国家的数据 + value = [d['IPv6'], d['DNSSEC'], d['DoH'], d['DoT']] + cou_data = { + "name": d['COU'], + 'title': "支持各类协议的解析器数量为：{IPv6：" + str(d['IPv6']) + "，DNSSEC：" + str( + d['DNSSEC']) + "，DoH：" + str(d['DoH']) + "，DoT：" + str(d['DoT']) + "}", + "value": d['R'] + } + cou_list.append(cou_data) + + return {"code": 200, "dataObject": {"earthAddTitle": cou_list}} + # 查询目标 + else: + # 查询目标，根据v4、v6地址分类 + res = da.get_data(data_type="target", + search={"ADDRv4": ip} if "." in str(ip) else {"ADDRv6": ip}) + # 支持协议 + proto = [] + for p in ["IPv6", "DNSSEC", "DOH", "DOT"]: + if res[0][p]: + proto.append(p) + target = [{"name": res[0]["COU"], + "title": "该解析器支持：" + '、'.join(proto) + " 协议", + "value": 1, }] + return {"code": 200, "dataObject": {"earthAddTitle": target}} -- cgit v1.2.3 From df5dfafc12281720faa68c57bcd4b966f559967c Mon Sep 17 00:00:00 2001 From: handingkang <18791985373@163.com> Date: Mon, 22 Apr 2024 17:35:38 +0800 Subject: 同步说明文档信息 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script.sql | 162 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ server/README.md | 24 ++++++++- 2 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 script.sql diff --git a/script.sql b/script.sql new file mode 100644 index 0000000..0830b72 --- /dev/null +++ b/script.sql @@ -0,0 +1,162 @@ +create table if not exists AGENT +( + AGENT_ID varchar(255) not null + primary key, + IPADDR varchar(255) null, + START_TIME datetime default CURRENT_TIMESTAMP null, + LAT varchar(255) null, + LNG varchar(255) null, + AGENT_TYPE varchar(255) null, + SYS varchar(255) null, + PORT int null, + CPU_NUM int null, + STATUS tinyint(1) null, + MEM varchar(255) null, + IDLE tinyint(1) null +) + charset = utf8mb3; + +create table if not exists POLICY +( + P_ID varchar(255) not null + primary key, + P_EXE varchar(255) null, + P_TYPE varchar(255) null, + P_DESC varchar(255) null, + P_PAYLOAD varchar(255) null, + P_NAME varchar(255) null, + P_PROTO varchar(255) null +) + charset = utf8mb3; + +create table if not exists TARGETDATA +( + TARGET_ID int auto_increment + primary key, + ADDRv4 varchar(255) not null, + ADDRv6 varchar(255) not null, + IPv6 tinyint(1) null, + DNSSEC tinyint(1) null, + DOT tinyint(1) null, + DOH tinyint(1) null, + COU varchar(255) null, + ISP varchar(255) null, + LAT varchar(255) null, + LNG varchar(255) null, + UPDATED_TIME datetime default CURRENT_TIMESTAMP null, + PROTECT varchar(255) null, + DOH_DOMAIN varchar(255) null, + constraint TARGETDATA_pk + unique (ADDRv4), + constraint TARGETDATA_pk_2 + unique (ADDRv6) +) + charset = utf8mb3; + +create table if not exists USER +( + USER_ID varchar(255) not null + primary key, + USER_NAME varchar(255) null, + CREATED_BY varchar(255) null, + CREATED_TIME datetime default CURRENT_TIMESTAMP null, + USER_PWD_HASH varchar(255) null, + USER_GROUP varchar(255) null +) + charset = utf8mb3; + +create table if not exists SYSLOG +( + LOG_ID int auto_increment + primary key, + LOG_LEVEL varchar(255) null, + LOG_INFO varchar(255) null, + LOG_TIME datetime default CURRENT_TIMESTAMP null, + S_IP varchar(255) null, + USER_ID varchar(255) null, + constraint SYSLOG_ibfk_1 + foreign key (USER_ID) references USER (USER_ID) +) + charset = utf8mb3; + +create index USER_ID + on SYSLOG (USER_ID); + +create table if not exists TASK +( + TASK_ID varchar(255) not null + primary key, + TASK_NAME varchar(255) null, + AGENT_ID varchar(255) null, + CREATED_BY varchar(255) null, + TARGET_IP varchar(255) null, + CREATED_TIME datetime default CURRENT_TIMESTAMP null, + POLICY int null, + STATUS varchar(255) null, + POLICY_DELAY varchar(255) null, + TASK_DELAY varchar(255) null, + TARGET_SCAN varchar(255) null, + TARGET_DOMAIN varchar(255) null, + TARGET_RTYPE varchar(255) null, + TARGET_RR varchar(255) null, + constraint TASK_ibfk_1 + foreign key (AGENT_ID) references AGENT (AGENT_ID), + constraint TASK_ibfk_2 + foreign key (CREATED_BY) references USER (USER_ID) +) + charset = utf8mb3; + +create index AGENT_ID + on TASK (AGENT_ID); + +create index CREATED_BY + on TASK (CREATED_BY); + +create table if not exists TASK_POLICY +( + TP_ID int auto_increment + primary key, + TP_TIME datetime default CURRENT_TIMESTAMP null, + POLICY varchar(255) null, + POLICY_PARAM varchar(255) null, + FOR_TASK varchar(255) null, + constraint TASK_POLICY_ibfk_1 + foreign key (POLICY) references POLICY (P_ID), + constraint TASK_POLICY_ibfk_2 + foreign key (FOR_TASK) references TASK (TASK_ID) +) + charset = utf8mb3; + +alter table TASK + add constraint fk_TASK_TASK_POLICY_1 + foreign key (POLICY) references TASK_POLICY (TP_ID); + +create table if not exists TASK_LOG +( + TLOG_ID int auto_increment + primary key, + CREATED_BY_AGENT varchar(255) null, + CREATED_TIME datetime default CURRENT_TIMESTAMP null, + TLOG_LEVEL varchar(255) null, + TLOG_INFO varchar(255) null, + TLOG_TP int null, + constraint TASK_LOG_ibfk_1 + foreign key (CREATED_BY_AGENT) references AGENT (AGENT_ID), + constraint TASK_LOG_ibfk_2 + foreign key (TLOG_TP) references TASK_POLICY (TP_ID) +) + charset = utf8mb3; + +create index CREATED_BY_AGENT + on TASK_LOG (CREATED_BY_AGENT); + +create index TLOG_TP + on TASK_LOG (TLOG_TP); + +create index FOR_TASK + on TASK_POLICY (FOR_TASK); + +create index POLICY + on TASK_POLICY (POLICY); + + diff --git a/server/README.md b/server/README.md index 4b8b3e7..f81b547 100644 --- a/server/README.md +++ b/server/README.md @@ -2,7 +2,11 @@ ## 简介 -主控端代码，运行app.py将启动一个flask服务，端口为2525。 +主控端代码，运行app.py将启动一个flask服务，默认端口为12525。 +包含三个分支： +main分支，稳定版本 +dev分支，用于根据fake分支完善的接口定义开发实际业务逻辑 +fake分支，用于快速定义新的数据接口与前端对接 访问/docs可以看到swagger-api文档 @@ -16,3 +20,21 @@ | target.py | 目标感知 | | task.py | 任务管理 | +model.py文件包含了一些参数与数据库表字段名或类型之间的映射关系 + +util.py 提供了访问数据库的工具类和日志输出函数 + +## 部署过程 + +1. 首先根据script.sql创建符合要求的数据库表，并在settings.py中配置对应的数据库连接选项 +2. 接下来使用pip 根据requirements.txt安装第三方环境依赖 + +```shell +pip install -r requirements.txt +``` + +3. 启动主控端服务 + +```shell +python app.py +``` \ No newline at end of file -- cgit v1.2.3