diff options
103 files changed, 1309 insertions, 7110 deletions
diff --git a/benchmark/ck_filter.json b/benchmark/ck_filter.json deleted file mode 100644 index 130744e5..00000000 --- a/benchmark/ck_filter.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "version": "1.0", - "name": "connection_record_log", - "namespace": "Clickhouse", - "filters": [ - { - "name":"@start", - "value": "'2021-01-11 10:00:00'" - }, - { - "name":"@end", - "value": "'2021-01-13 11:00:00'" - }, - { - "name":"@common_filter", - "value": [ - "common_client_ip='192.168.44.21'and common_server_port=443", - "common_server_ip='8.8.8.8' and common_server_port=443", - "http_domain like '%baidu%' and common_server_port=443", - "http_domain like '%baidu%'and common_server_port=443" - ] - }, - { - "name":"@index_filter", - "value": [ - "common_server_ip='8.8.8.8'", - "common_client_ip='192.168.44.21'", - "http_domain like '%baidu%'", - "common_sled_ip='%192.168%'", - "common_subscriber_id='%test%'" - ] - } - ] -}
\ No newline at end of file diff --git a/benchmark/ck_queries_template.sql b/benchmark/ck_queries_template.sql deleted file mode 100644 index fc5fb926..00000000 --- a/benchmark/ck_queries_template.sql +++ /dev/null @@ -1,108 +0,0 @@ ---Q01.Count(1) -select count(1) from connection_record_log ---Q02.All Fields Query (default) -SELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) LIMIT 30 ---Q03.All Fields Query order by Time desc -SELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ---Q04.All Fields Query order by Time asc -SELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time asc LIMIT 30 ---Q05.All Fields Query by Filter -SELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30 ---Q06.Default Fields Query by Filter -SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30 ---Q07.All Fields Query (sub query by time) -SELECT * FROM connection_record_log AS connection_record_log WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ---Q08.All Fields Query (sub query by log id) -SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ---Q09.Default Field Query (sub query by time) -SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ---Q10.Default Field Query (sub query by log id) -SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( select common_log_id FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)) ORDER BY common_recv_time DESC LIMIT 30 ---Q11.Default Field Query by Server IP (sub query by log id with Index Table) -SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_common_server_ip AS connection_record_log_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 ---Q12.Default Field Query by Client IP (sub query by log id with Index Table) -SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_common_client_ip AS connection_record_log_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 ---Q13.Default Field Query by Domain (sub query by log id with Index Table) -SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_http_domain AS connection_record_log_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 ---Q14.Default Field Query by Subscriber ID (sub query by log id with Index Table) -SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_common_subscriber_id AS connection_record_log_common_subscriber_id WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 ---Q15.All Fields Query by Client IP (sub query by log id with index Table) -SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_common_client_ip AS connection_record_log_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 ---Q16.All Fields Query by Server IP(sub query by log id with index Table) -SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_common_server_ip AS connection_record_log_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 ---Q17.All Fields Query by Domain(sub query by log id with index Table) -SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_http_domain AS connection_record_log_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 ---Q18.All Fields Query by Subscriber ID(sub query by log id with index Table) -SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_common_subscriber_id AS connection_record_log_common_subscriber_id WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 ---Q19.Session Logs Sent to Database Trend(Time Grain 5 minute) -SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", count(common_log_id) AS "logs" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ) ) GROUP BY "Receive Time" LIMIT 100000 ---Q20.Traffic Bandwidth Trend(Time Grain 30 second) -SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 30 SECOND)))) AS stat_time, sum(common_c2s_byte_num) AS bytes_sent, sum(common_s2c_byte_num) AS bytes_received, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets, sum(common_sessions) AS sessions FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time ORDER BY stat_time ASC LIMIT 100000 ---Q21.Log Tend by Type (Time Grain 5 minute) -SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) GROUP BY stat_time, common_schema_type ORDER BY stat_time ASC LIMIT 100000 ---Q22.Traffic Metrics Analytic -SELECT round(sum(common_s2c_byte_num) * 8 / 300,2) AS trafficInBits, round(sum(common_c2s_byte_num) * 8 / 300,2) AS trafficOutBits, round(sum(common_s2c_byte_num + common_c2s_byte_num) * 8 / 300,2) AS trafficTotalBits, round(sum(common_s2c_pkt_num) / 300,2) AS trafficInPackets, round(sum(common_c2s_pkt_num) / 300,2) AS trafficOutPackets, round(sum(common_s2c_pkt_num + common_c2s_pkt_num) / 300,2) AS trafficTotalPackets, round(sum(common_sessions) / 300,2) AS sessions FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ---Q23.Traffic Endpoints Metrics Trend(Time Grain 5 minute) -SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", uniq(common_internal_ip) AS "Unique Internal IP", uniq(common_external_ip) AS "Unique External IP", uniq(common_subscriber_id) AS "Unique Subscriber ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 100000 ---Q24.Endpoint Unique Num by L4 Protocol -SELECT 'all' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) UNION ALL SELECT 'tcp' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) UNION ALL SELECT 'UDP' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_UDP', 'IPv6_UDP' ) ---Q25.One-sided Connection Trend(Time Grain 5 minute) -SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, (CASE WHEN common_stream_dir = 1 THEN 'c2s' WHEN common_stream_dir = 2 THEN 's2c' WHEN common_stream_dir = 3 THEN 'double' ELSE 'None' END) AS type, sum(common_sessions) AS sessions FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time, common_stream_dir ORDER BY stat_time ASC LIMIT 100000 ---Q26. Estimated One-sided Sessions with Bandwidth -SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_sessions) AS "sessions", sum(if(common_stream_dir <> 3, common_sessions, 0)) AS "one_side_sessions", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", round(one_side_sessions / sessions, 2) AS one_side_percent FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 100000 ---Q27.Estimated TCP Sequence Gap Loss -SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", sum(common_c2s_tcp_lostlen + common_s2c_tcp_lostlen) AS "gap_loss_bytes", round(gap_loss_bytes / bytes, 2) AS gap_loss_percent FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Receive Time" LIMIT 100000 ---Q28.Top30 Server IP by Bytes -SELECT "server_ip" AS "server_ip" , SUM(coalesce("bytes",0)) AS "bytes" , SUM(coalesce("bytes_sent",0)) AS "Sent" , SUM(coalesce("bytes_received",0)) AS "Received" , SUM(coalesce("sessions",0)) AS "sessions" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(common_c2s_byte_num+common_s2c_byte_num) AS "bytes" , SUM(coalesce(common_sessions,0)) AS "sessions" , common_server_ip AS "server_ip" FROM connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" desc ) GROUP BY "server_ip" ORDER BY "bytes" desc LIMIT 30 ---Q29.Top30 Client IP by Sessions -SELECT common_client_ip , COUNT(*) AS sessions FROM connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_client_ip ORDER BY sessions desc LIMIT 0,30 ---Q30.Top30 TCP Server Ports by Sessions -SELECT "Server Port" AS "Server Port", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_port AS "Server Port", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Server Port" LIMIT 1048576) GROUP BY "Server Port" ORDER BY "Sessions" DESC LIMIT 30 ---Q31.Top30 Domian by Bytes -SELECT "domain" AS "Website Domain" , SUM(coalesce("bytes",0)) AS "Throughput" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "bytes" , http_domain AS "domain" FROM connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "domain" ORDER BY "bytes" desc ) GROUP BY "domain" ORDER BY "Throughput" desc LIMIT 30 ---Q32.Top30 Endpoint Devices by Bandwidth -SELECT "device_id" AS "device_id", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, common_device_id AS "device_id" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 30 ---Q33.Top30 Domain by Unique Client IP -SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Client IP", 0)) AS "Client IP" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Client IP" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 30 ---Q34.Top100 Most Time Consuming Domains -SELECT "Domain" AS "Domain", avg(coalesce("Avg Establish Latency(ms)", 0)) AS "Avg Establish Latency(ms)" FROM (SELECT http_domain AS "Domain", avg(coalesce(common_establish_latency_ms, 0)) AS "Avg Establish Latency(ms)" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Avg Establish Latency(ms)" DESC LIMIT 100 ---Q35.Top30 Sources by Sessions -SELECT "source" AS "source", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(common_subscriber_id, ''), nullif(common_client_ip, '')) AS "source", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 30 ---Q36.Top30 Destinations by Sessions -SELECT "destination" AS "destination", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(http_domain, ''), nullif(common_server_ip, '')) AS "destination", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 30 ---Q37.Top30 Destination Regions by Bandwidth -SELECT "server_location" AS "server_location", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT arrayElement(splitByString(',', common_server_location), length(splitByString(',', common_server_location))) AS "server_location", sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "bytes", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 30 ---Q38.Top30 URLS by Sessions -SELECT "Http URL" AS "Http URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "Http URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http URL" LIMIT 1048576) GROUP BY "Http URL" ORDER BY "Sessions" DESC LIMIT 30 ---Q39.Top30 Destination Transmission APP by Bandwidth -SELECT "server_ip" AS "server_ip", groupUniqArray(coalesce("trans_app", 0)) AS "trans_app", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", groupUniqArray(concat(common_l4_protocol, '/', toString(common_server_port))) AS "trans_app", common_server_ip AS "server_ip" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 30 ---Q40.Browsing Users by Website domains and Sessions -SELECT "Subscriber ID" AS "Subscriber ID", "Http.Domain" AS "Http.Domain", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT http_domain AS "Http.Domain", common_subscriber_id AS "Subscriber ID", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) AND notEmpty(common_subscriber_id) ) GROUP BY "Http.Domain", "Subscriber ID" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Subscriber ID", "Http.Domain" ORDER BY "sessions" DESC LIMIT 100000 ---Q41.Top Domain and Server IP by Bytes Sent -SELECT "Http.Domain" AS "Http.Domain" , "Server IP" AS "Server IP" , SUM(coalesce("Bytes Sent",0)) AS "Bytes Sent" FROM ( SELECT common_server_ip AS "Server IP" , http_domain AS "Http.Domain" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "Bytes" , SUM(coalesce(common_c2s_byte_num,0)) AS "Bytes Sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "Bytes Received" FROM connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "Server IP" , "Http.Domain" ORDER BY "Bytes" desc LIMIT 1048576 ) GROUP BY "Http.Domain" , "Server IP" ORDER BY "Bytes Sent" desc LIMIT 100000 ---Q42.Top30 Website Domains by Client IP and Sessions -SELECT "Http.Domain" AS "Http.Domain", "Client IP" AS "Client IP", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT common_client_ip AS "Client IP", http_domain AS "Http.Domain", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Client IP", "Http.Domain" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Http.Domain", "Client IP" ORDER BY "sessions" DESC LIMIT 100000 ---Q43.Domain is Accessed by Unique Client IP Trend(bytes Time Grain 5 minute) -SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , http_domain AS Domain, COUNT(DISTINCT(common_client_ip)) AS nums FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) AND http_domain IN ( SELECT http_domain FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY SUM(common_s2c_byte_num+common_c2s_byte_num) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , http_domain ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 100000 ---Q44. Domain is Accessed by Unique Client IP Trend(sessions,Time Grain 5 minute) -SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),3600)*3600) AS stat_time , http_domain , uniq (common_client_ip) AS nums FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start)-604800 AND common_recv_time < toDateTime(@end) AND http_domain IN ( SELECT http_domain FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY COUNT(*) desc LIMIT 5 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 3600)*3600), http_domain ORDER BY stat_time desc LIMIT 100000 ---Q45.Bandwidth Trend with Device ID(Time Grain 5 minute) -SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", common_device_id AS "Device ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time", "Device ID" LIMIT 100000 ---Q46.Internal IP by Sled IP and Sessions -SELECT "Internal IP" AS "Internal IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_internal_ip AS "Internal IP", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Sled IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 100000 ---Q47.Bandwidth Trend with Internal IP (Time Grain 5 minute) -SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_c2s_pkt_num + common_s2c_pkt_num, 0)) AS "Packets", sum(coalesce(common_sessions, 0)) AS "New Sessions", sum(coalesce(common_c2s_byte_num, 0)) AS "Bytes Sent", sum(coalesce(common_s2c_byte_num, 0)) AS "Bytes Received", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent", sum(coalesce(common_s2c_pkt_num, 0)) AS "Packets Received" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) GROUP BY "Receive Time" LIMIT 100000 ---Q48.Top30 Domains Detail with Internal IP -SELECT "Domain" AS "Domain", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_domain AS "Domain", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Sessions" DESC LIMIT 30 ---Q49.Top30 URLS Detail with Internal IP -SELECT "URL" AS "URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_url) ) GROUP BY "URL" LIMIT 1048576) GROUP BY "URL" ORDER BY "Sessions" DESC LIMIT 30 ---Q50.Top Domains with Unique Client IP and Subscriber ID -SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Unique Client IP", 0)) AS "Unique Client IP", sum(coalesce("Unique Subscriber ID", 0)) AS "Unique Subscriber ID" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Unique Client IP", uniq(common_subscriber_id) AS "Unique Subscriber ID" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Unique Client IP" DESC LIMIT 100 ---Q51.Top100 Domains by Packets sent -SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Packets Sent", 0)) AS "Packets Sent" FROM (SELECT http_domain AS "Http.Domain", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Packets Sent" DESC LIMIT 100 ---Q52.Internal and External asymmetric traffic -SELECT "Internal IP" AS "Internal IP", "External IP" AS "External IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_external_ip AS "External IP", common_internal_ip AS "Internal IP", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes Sent+Bytes Received", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Sled IP", "External IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "External IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 500 ---Q53.Client and Server ASN asymmetric traffic -SELECT "Client ASN" AS "Client ASN", "Server ASN" AS "Server ASN", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_asn AS "Server ASN", common_client_asn AS "Client ASN", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Server ASN", "Client ASN" LIMIT 1048576) GROUP BY "Client ASN", "Server ASN" ORDER BY "Sessions" DESC LIMIT 500 ---Q54.Top handshake latency by Website and Client IPs -SELECT "SSL.SNI" AS "SSL.SNI", "Client IP" AS "Client IP", avg(coalesce("Establish Latency(ms)", 0)) AS "Establish Latency(ms)" FROM (SELECT common_client_ip AS "Client IP", ssl_sni AS "SSL.SNI", avg(coalesce(common_establish_latency_ms, 0)) AS "Establish Latency(ms)" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Client IP", "SSL.SNI" LIMIT 1048576) GROUP BY "SSL.SNI", "Client IP" ORDER BY "Establish Latency(ms)" DESC LIMIT 500
\ No newline at end of file diff --git a/benchmark/druid_filter.json b/benchmark/druid_filter.json deleted file mode 100644 index b8750295..00000000 --- a/benchmark/druid_filter.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "version": "1.0", - "name": "connection_record_log", - "namespace": "druid", - "filters": [ - { - "name":"@start", - "value": "'2021-01-11 10:00:00'" - }, - { - "name":"@end", - "value": "'2021-01-13 11:00:00'" - }, - { - "name":"@common_filter", - "value": [ - - ] - } - ] -}
\ No newline at end of file diff --git a/benchmark/druid_queries_template.sql b/benchmark/druid_queries_template.sql deleted file mode 100644 index 8ac4b997..00000000 --- a/benchmark/druid_queries_template.sql +++ /dev/null @@ -1,130 +0,0 @@ ---Q01.某策略命中计数 -select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end and policy_id=40 group by policy_id ---Q02. -select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end and policy_id in (9,10,88,45) group by policy_id ---Q03.某策略命中计数趋势 -select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from security_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end and policy_id=10 group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 100 ---Q04.某策略命中计数趋势 -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, sum(hits) as hits from security_event_hits_log where __time >= @start and __time < @end and policy_id=10 group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') limit 100 ---Q05.某策略命中时间（首次和最近一次) -select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from security_event_hits_log where policy_id in (100,101 ,105, 102) group by policy_id ---Q06. -select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from security_event_hits_log where policy_id in (100,101 ,105, 102) group by policy_id ---Q07.TopN 命中策略 -select policy_id, action, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id, action order by hits desc limit 200 ---Q08. -select policy_id, action, sum(hits) as hits from security_event_hits_log where __time >=@start and __time <@end group by policy_id, action order by hits desc limit 200 ---Q09.代理策略命中统计 1. 某策略命中计数 -select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end and policy_id=100 group by policy_id ---Q10. -select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end and policy_id=100 group by policy_id ---Q11.某策略命中趋势 -select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end and policy_id=100 group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 101 ---Q12. -select FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end and policy_id=100 group by FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) limit 101 ---Q13.某策略命中时间（首次和最近一次 -select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from proxy_event_hits_log where policy_id in (100,101,102,105) group by policy_id ---Q14. -select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from proxy_event_hits_log where policy_id in (100,101,102,105) group by policy_id ---Q15. TopN 命中策略 -select policy_id, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id, sub_action order by hits desc limit 200 ---Q16. TopN 命中策略 -select policy_id, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id, sub_action order by hits desc limit 200 ---Q17. Proxy 操纵动作命中计数 -select sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by sub_action ---Q18. -select sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end group by sub_action ---Q19. Proxy 操纵动作命中趋势 -select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') , sub_action limit 100 ---Q20. -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s'), sub_action limit 100 ---Q21. Proxy Pinning TIMESTAMP 计数 -SELECT sum(not_pinning_num) AS sessions, 'notPinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(pinning_num) AS sessions, 'pinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(maybe_pinning_num) AS sessions, 'maybePinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end ---Q22. Proxy Pinning计数趋势 -SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 100 ---Q23. -SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(not_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time>= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 100 ---Q24. -SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(maybe_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 100 ---Q25.Traffics-带宽统计 1. Traffic IN/OUT 计数 Bytes -select sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end ---Q26. -select sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >= @start and __time < @end ---Q27. //Packets -select sum(total_in_packets) as traffic_in_packets, sum(total_out_packets) as traffic_out_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end ---Q28. -select sum(total_in_packets) as traffic_in_packets, sum(total_out_packets) as traffic_out_packets from traffic_metrics_log where __time >= @start and __time < @end ---Q29.//Sessions -select sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end ---Q30. -select sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end ---Q31.2. Traffic IN/OUT 带宽趋势 //Bytes -select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_bytes' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') ---Q32. -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_out_bytes' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') ---Q33. Packets -select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') ---Q34. -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') ---Q35. Packets -select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') ---Q36. -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') ---Q37.Sessions -select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') ---Q38. -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') ---Q39.四、 Traffics-计数统计 1. 新建与活跃链接计数 -select sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end ---Q40. -select sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >= @start and __time < @end ---Q41. 2.新建与活跃链接计数趋势 -select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'established_conn_num' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') ---Q42. -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'established_conn_num' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') ---Q43. 3. 安全策略动作计数 Bytes -select sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end ---Q44. -select sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >= @start and __time < @end ---Q45.Packets -select sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end ---Q46. -select sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >= @start and __time < @end ---Q47.Sessions -select sum(default_conn_num) as default_sessions, sum(allow_conn_num) as allow_sessions, sum(deny_conn_num) as deny_sessions, sum(monitor_conn_num) as monitor_sessions, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end ---Q48. -select sum(default_conn_num) as default_sessions, sum(allow_conn_num) as allow_sessions, sum(deny_conn_num) as deny_sessions, sum(monitor_conn_num) as monitor_sessions, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >= @start and __time < @end ---Q49.4. 安全策略动作趋势 Bytes -select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_bytes' as type, sum(default_in_bytes+default_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_bytes' as type, sum(allow_in_bytes+allow_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_bytes' as type, sum(deny_in_bytes+deny_out_bytes) as bytes from traffic_metrics_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_bytes' as type, sum(monitor_in_bytes+monitor_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_bytes' as type, sum(intercept_in_bytes+intercept_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') ---Q50. -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'default_bytes' as type, sum(default_in_bytes+default_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'allow_bytes' as type, sum(allow_in_bytes+allow_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'deny_bytes' as type, sum(deny_in_bytes+deny_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'monitor_bytes' as type, sum(monitor_in_bytes+monitor_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'intercept_bytes' as type, sum(intercept_in_bytes+intercept_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') ---Q51.packets -select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_packets' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_packets' as type, sum(allow_in_packets+allow_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_packets' as type, sum(deny_in_packets+deny_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_packets' as type, sum(monitor_in_packets+monitor_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_packets' as type, sum(intercept_in_packets+intercept_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') ---Q52. -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'default_packets' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'allow_packets' as type, sum(allow_in_packets+allow_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'deny_packets' as type, sum(deny_in_packets+deny_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'monitor_packets' as type, sum(monitor_in_packets+monitor_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'intercept_packets' as type, sum(intercept_in_packets+intercept_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') ---Q53.sessions -select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_conn_num' as type, sum(default_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_conn_num' as type, sum(allow_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_conn_num' as type, sum(deny_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_conn_num' as type, sum(monitor_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_conn_num' as type, sum(intercept_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') ---Q54. -select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'default_conn_num' as type, sum(default_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'allow_conn_num' as type, sum(allow_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'deny_conn_num' as type, sum(deny_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'monitor_conn_num' as type, sum(monitor_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'intercept_conn_num' as type, sum(intercept_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') ---Q55.五. Traffics-TOPN统计 1. TopN 源IP -select source as client_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by=source group by source order by source desc limit 100 ---Q56. -select source as client_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >= @start and __time < @end and order_by=source group by source order by source desc limit 100 ---Q57. 2. TopN 目的IP -select destination as server_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_external_host_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by=destination group by destination order by destination desc limit 100 ---Q58. -select destination as server_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_external_host_log where __time >= @start and __time < @end and order_by=destination group by destination order by destination desc limit 100 ---Q59. 3. TopN 域名 -select domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by=domain group by domain order by domain desc limit 200 ---Q60. -select domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >= @start and __time < @end and order_by=domain group by domain order by domain desc limit 200 ---Q61.4.TopN 用户 -select subscriber_id, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_user_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by=subscriber_id group by subscriber_id order by subscriber_id desc limit 200 ---Q62. -select subscriber_id, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_user_log where __time >= @start and __time < @end and order_by=subscriber_id group by subscriber_id order by subscriber_id desc limit 200 ---Q63.六、命中URL统计 -select url,sum(session_num) as hits from top_urls_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by url order by hits desc limit 100 ---Q64. -select url,sum(session_num) as hits from top_urls_log where __time >= @start and __time < @end group by url order by hits desc limit 100 ---Q65.劫持客户端数量 -SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end and policy_id=0 group by policy_id diff --git a/benchmark/engine_filter.json b/benchmark/engine_filter.json deleted file mode 100644 index b5816cdd..00000000 --- a/benchmark/engine_filter.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "version": "1.0", - "name": "connection_record_log", - "namespace": "Engine", - "filters": [ - { - "name":"@start", - "value": "'2021-01-11 10:00:00'" - }, - { - "name":"@end", - "value": "'2021-01-13 11:00:00'" - }, - { - "name":"@common_filter", - "value": [ - "common_client_ip='192.168.44.21'and common_server_port=443", - "common_server_ip='8.8.8.8' and common_server_port=443", - "http_domain like '%baidu%' and common_server_port=443", - "http_domain like '%baidu%'and common_server_port=443" - ] - } - ] -}
\ No newline at end of file diff --git a/benchmark/engine_queries_template.sql b/benchmark/engine_queries_template.sql deleted file mode 100644 index 598109b5..00000000 --- a/benchmark/engine_queries_template.sql +++ /dev/null @@ -1,62 +0,0 @@ ---Q01. -select toDateTime(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, ssl_sni, ssl_version from security_event_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) and common_client_ip like '49.7%' order by common_recv_time desc limit 0,20 ---Q02. -select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, ssl_sni, ssl_version from security_event_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) and common_client_ip like '49.7%' order by common_recv_time desc limit 0,20 ---Q03. -select toDateTime(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, http_host,http_url,http_user_agent from proxy_event_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) and http_host like '%joy.cn%' order by common_recv_time desc limit 0,20 ---Q04. -select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, http_host,http_url,http_user_agent from proxy_event_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) and http_host like '%joy.cn%' order by common_recv_time desc limit 0,20 ---Q05. -select toDateTime(common_recv_time) as common_recv_time, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, radius_framed_ip, radius_account from radius_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@start) and radius_account='T1yRd' order by common_recv_time desc limit 0,20 ---Q06. -select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, http_host,http_url,http_user_agent from proxy_event_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) and http_host like '%joy.cn%' order by common_recv_time desc limit 0,20 ---Q07. -select toDateTime(common_recv_time) as common_recv_time, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, radius_framed_ip, radius_account from radius_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@start) and radius_account='T1yRd' order by common_recv_time desc limit 0,20 ---Q08. -select FROM_UNIXTIME(common_recv_time) as common_recv_time, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, radius_framed_ip, radius_account from radius_record_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) and radius_account='T1yRd' order by common_recv_time desc limit 0,20 ---Q09. -select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_log_id,common_policy_id,common_subscriber_id,common_client_ip,common_client_port,common_l4_protocol,common_address_type,common_server_ip,common_server_port,common_action,common_direction,common_sled_ip,common_client_location,common_client_asn,common_server_location,common_server_asn,common_c2s_pkt_num,common_s2c_pkt_num,common_c2s_byte_num,common_s2c_byte_num,common_schema_type,common_sub_action,common_device_id, FROM_UNIXTIME(common_start_time) as common_start_time, FROM_UNIXTIME(common_end_time) as common_end_time,common_establish_latency_ms,common_con_duration_ms,common_stream_dir,common_stream_trace_id,http_url,http_host,http_domain,http_request_body,http_response_body,http_cookie,http_referer,http_user_agent,http_content_length,http_content_type,http_set_cookie,http_version,http_response_lantency_ms,http_action_file_size,http_session_duration_ms,mail_protocol_type,mail_account,mail_from_cmd,mail_to_cmd,mail_from,mail_to,mail_cc,mail_bcc,mail_subject,mail_attachment_name,mail_eml_file,dns_message_id,dns_qr,dns_opcode,dns_aa,dns_tc,dns_rd,dns_ra,dns_rcode,dns_qdcount,dns_ancount,dns_nscount,dns_arcount,dns_qname,dns_qtype,dns_qclass,dns_cname,dns_sub,dns_rr,ssl_sni,ssl_san,ssl_cn,ssl_pinningst,ssl_intercept_state,ssl_server_side_latency,ssl_client_side_latency,ssl_server_side_version,ssl_client_side_version,ssl_cert_verify,ssl_error,quic_version,quic_sni,quic_user_agent,ftp_account,ftp_url,ftp_content from security_event_log where common_recv_time >= @start and common_recv_time < @end and common_log_id in ( 289451559826124800 , 332 ) and ( common_server_ip like '192.168%' ) order by common_recv_time desc limit 100000 ---Q10. -select count(1) from connection_record_log ---Q11. -select count(*) from connection_record_log ---Q12. -select * from tsg_galaxy_v3.connection_record_log limit 1 ---Q13.二次查询 -select * from (select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','zero')) as stat_time from connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) group by stat_time order by stat_time asc) ---Q14.radius账户申请客户端IP变化 -select framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >=toDateTime(@start) and event_timestamp <toDateTime(@end) and account='admin' group by framed_ip ---Q15. -select framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >=@start and event_timestamp < @end and account='admin' group by framed_ip ---Q16.用户IP承载用户变化 -select account, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >= @start and event_timestamp < @end and framed_ip='127.0.0.1' group by account ---Q17. -select account, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >= @start and event_timestamp < @end and framed_ip='127.0.0.1' group by account ---Q18.预置Internal Hosts 报告 -select common_client_ip, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@end)) group by common_client_ip order by sessions desc limit 0,100 ---Q19.预置External Hosts 报告 -select common_server_ip, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by common_server_ip order by sessions desc limit 0,100 ---Q20.预置Domains报告 -select http_domain AS domain,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(domain) GROUP BY domain ORDER BY bytes DESC LIMIT 100 ---Q21. -select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_domain, uniq (common_client_ip) as nums from connection_record_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_domain in (select http_domain from connection_record_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_domain) group by http_domain order by SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_domain order by stat_time asc limit 500 ---Q22. -SELECT http_host as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) GROUP BY host ORDER BY bytes DESC union all SELECT 'totals' as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes, SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes, SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) ---Q23.预置HTTP/HTTPS URLS报告 -SELECT http_url AS url,count(*) AS sessions FROM proxy_event_log WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) GROUP BY url ORDER BY sessions DESC LIMIT 100 ---Q24. -select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_url, count(distinct(common_client_ip)) as nums from proxy_event_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_url IN (select http_url from proxy_event_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) group by http_url order by count(*) desc limit 10 )group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_url order by stat_time asc limit 500 ---Q25. -select common_subscriber_id as user, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) group by common_subscriber_id order by sessions desc limit 0,100 ---Q26. -SELECT common_subscriber_id as user,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) GROUP BY user ORDER BY bytes DESC LIMIT 100 ---Q27.RADIUS账户总计 -select count(distinct(framed_ip)) as active_ip_num , sum(acct_session_time) as online_duration from (select any(framed_ip) as framed_ip ,max(acct_session_time) as acct_session_time from radius_onff_log where account='000jS' and event_timestamp >= @start and event_timestamp < @end group by acct_session_id) ---Q28.RADIUS账户IP详情 -select distinct(framed_ip) as framed_ip from radius_onff_log where account='000iS' and event_timestamp >= @start and event_timestamp < @end ---Q29.RADIUS账户访问详情 -select max(if(acct_status_type=1,event_timestamp,0)) as start_time,max(if(acct_status_type=2,event_timestamp,0)) as end_time, any(framed_ip) as ip,max(acct_session_time) as online_duration from radius_onff_log where account='000jS' and event_timestamp >= @start and event_timestamp < @end group by acct_session_id order by start_time desc limit 200 ---Q30.自定义函数测试 -SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end and policy_id=0 group by policy_id ---Q31. -select TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','previous') as stat_time from connection_record_log where common_recv_time > @start and common_recv_time < @end group by stat_time
\ No newline at end of file diff --git a/config/application-dev.yml b/config/application-dev.yml deleted file mode 100644 index e176bef8..00000000 --- a/config/application-dev.yml +++ /dev/null @@ -1,78 +0,0 @@ -############################################################################### -## Dev Environment -############################################################################### -## Log configuration Path -logging: - config: ./config/log4j2-dev.xml -#MySql configuration -spring: - datasource: - driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://192.168.44.3:3306/tsg-bifang?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai - username: root - password: 111111 -## ClickhHouse configuration -clickhouse: - url: jdbc:clickhouse://192.168.44.12:8123 - driverClassName: ru.yandex.clickhouse.ClickHouseDriver - dbname: tsg_galaxy_v3 - realTimeAccount: - username: tsg_query - userKey: ceiec2018 - socketTimeOut: 60000 - longTermAccount: - username: tsg_report - userKey: ceiec2019 - socketTimeOut: 21700000 -############################################################################### -## Druid configuration -druid: - url: 192.168.44.12:8082/druid/v2/sql - dbname: druid - socketTimeOut: 60000 -############################################################################### -## Hbase configuration -hbase: - zookeeperQuorum: 192.168.44.12 - zookeeperPropertyClientPort: 2181 - zookeeperZnodeParent: /hbase - dbname: tsg - tableName: report_result - columnFamily: response - columnName: result - clientIpcPoolSize: 60 - rpcTimeout: 60000 -## xxl-job-admin configuration -xxl-job-admin: - url: http://192.168.44.12:8181/xxl-job-admin - userName: query - password: ceiec2018 -############################################################################### -## Engine configuration -engine: - maxCacheNum: 1048576 - defaultResultNum: 100000 -############################################################################### -## ArangoDB configuration -arango: - server: http://192.168.44.12:8529 - database: tsg_galaxy_v3 - username: query - password: ceiec2018 - jwturl: ${arango.server}/_db/${arango.database}/_open/auth - queryurl: ${arango.server}/_db/${arango.database}/_api/cursor - maxrows: 10000 - socketTimeOut: 300000 -## http pool config -http: - pool: - connect: - timeout: 30000 - max: - connection: 500 - per: - route: 200 - request: - timeout: 10000 - response: - timeout: 10000
\ No newline at end of file diff --git a/config/application-prod.yml b/config/application-prod.yml deleted file mode 100644 index d2760613..00000000 --- a/config/application-prod.yml +++ /dev/null @@ -1,78 +0,0 @@ -############################################################################### -## Prod Environment -############################################################################### -## Log configuration Path -logging: - config: ./config/log4j2-prod.xml -#MySql configuration -spring: - datasource: - driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://192.168.44.3:3306/tsg-bifang?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai - username: root - password: 111111 -## ClickhHouse configuration -clickhouse: - url: jdbc:clickhouse://192.168.44.12:8123 - driverClassName: ru.yandex.clickhouse.ClickHouseDriver - dbname: tsg_galaxy_v3 - realTimeAccount: - username: tsg_query - userKey: ceiec2018 - socketTimeOut: 60000 - longTermAccount: - username: tsg_report - userKey: ceiec2019 - socketTimeOut: 21700000 -############################################################################### -## Druid configuration -druid: - url: 192.168.44.12:8082/druid/v2/sql - dbname: druid - socketTimeOut: 60000 -############################################################################### -## Hbase configuration -hbase: - zookeeperQuorum: 192.168.44.12 - zookeeperPropertyClientPort: 2181 - zookeeperZnodeParent: /hbase - dbname: tsg - tableName: report_result - columnFamily: response - columnName: result - clientIpcPoolSize: 60 - rpcTimeout: 60000 -## xxl-job-admin configuration -xxl-job-admin: - url: http://192.168.44.12:8181/xxl-job-admin - userName: query - password: ceiec2018 -############################################################################### -## Engine configuration -engine: - maxCacheNum: 1048576 - defaultResultNum: 100000 -############################################################################### -## ArangoDB configuration -arango: - server: http://192.168.44.12:8529 - database: tsg_galaxy_v3 - username: query - password: ceiec2018 - jwturl: ${arango.server}/_db/${arango.database}/_open/auth - queryurl: ${arango.server}/_db/${arango.database}/_api/cursor - maxrows: 10000 - socketTimeOut: 300000 -## http pool config -http: - pool: - connect: - timeout: 30000 - max: - connection: 500 - per: - route: 200 - request: - timeout: 10000 - response: - timeout: 10000
\ No newline at end of file diff --git a/config/application-test.yml b/config/application-test.yml deleted file mode 100644 index b9a1148b..00000000 --- a/config/application-test.yml +++ /dev/null @@ -1,78 +0,0 @@ -############################################################################### -## Test Environment -############################################################################### -## Log configuration Path -logging: - config: ./config/log4j2-test.xml -## MySql configuration -spring: - datasource: - driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://192.168.44.3:3306/tsg-bifang?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai - username: root - password: 111111 -## ClickhHouse configuration -clickhouse: - url: jdbc:clickhouse://192.168.44.12:8123 - driverClassName: ru.yandex.clickhouse.ClickHouseDriver - dbname: tsg_galaxy_v3 - realTimeAccount: - username: tsg_query - userKey: ceiec2018 - socketTimeOut: 60000 - longTermAccount: - username: tsg_report - userKey: ceiec2019 - socketTimeOut: 21700000 -############################################################################### -## Druid configuration -druid: - url: 192.168.44.12:8082/druid/v2/sql - dbname: druid - socketTimeOut: 60000 -############################################################################### -## Hbase configuration -hbase: - zookeeperQuorum: 192.168.44.12 - zookeeperPropertyClientPort: 2181 - zookeeperZnodeParent: /hbase - dbname: tsg - tableName: report_result - columnFamily: response - columnName: result - clientIpcPoolSize: 60 - rpcTimeout: 60000 -## xxl-job-admin configuration -xxl-job-admin: - url: http://192.168.44.12:8181/xxl-job-admin - userName: query - password: ceiec2018 -############################################################################### -## Engine configuration -engine: - maxCacheNum: 1048576 - defaultResultNum: 100000 -############################################################################### -## ArangoDB configuration -arango: - server: http://192.168.44.12:8529 - database: tsg_galaxy_v3 - username: query - password: ceiec2018 - jwturl: ${arango.server}/_db/${arango.database}/_open/auth - queryurl: ${arango.server}/_db/${arango.database}/_api/cursor - maxrows: 10000 - socketTimeOut: 300000 -## http pool config -http: - pool: - connect: - timeout: 30000 - max: - connection: 500 - per: - route: 200 - request: - timeout: 10000 - response: - timeout: 10000
\ No newline at end of file diff --git a/config/application.yml b/config/application.yml index 4acf48d2..0ff47926 100644 --- a/config/application.yml +++ b/config/application.yml @@ -1,55 +1,22 @@ -############################################################################### -## springboot的配置文件顺序： -## 1. @TestPropertySource 注解 -## 2. 命令行参数 -## 3. Java系统属性（System.getProperties()） -## 4. 操作系统环境变量 -## 5. 只有在random.*里包含的属性会产生一个RandomValuePropertySource -## 6. 在打包的jar外的应用程序配置文件（application.properties，包含YML和profile变量） -## 7. 在打包的jar内的应用程序配置文件（application.properties，包含YML和profile变量） -## 8. 在@Configuration类上的@PropertySource注解 -## 9. 默认属性（使用SpringApplication.setDefaultProperties指定） -############################################################################### -## 服务配置 -server: - port: 9999 +nacos: + config: + type: yaml #配置集的配置格式 + server-addr: 192.168.44.12:8848 #配置中心地址 + namespace: nacos-namespace-galaxy #命名空间 + data-id: galaxy-qgw-service # 数据集ID + auto-refresh: true #开启自动刷新 + group: DEFAULT_GROUP #配置对应的分组 + username: galaxy #Nacos认证用户 + password: galaxy #Nacos认证密码 + bootstrap: + enable: true #开启配置预加载功能 + log: + enable: true #开启Nacos支持日志级别的加载时机 spring: profiles: active: dev -############################################################################### -## 项目参数，从maven中读取 -project: - name: galaxy-qgw-service - description: 统一数据查询网关 - version: 1.1 - groupId: com.mesalab - artifactId: galaxy-qgw-service - basedir: - core-package: com.mesalab.common - service-package: com.mesalab.qgw - author: - name: darnell - url: - email: -############################################################################### -management: - metrics: - tags: - application: ${project.name} - endpoint: - health: - show-details: always - shutdown: - enabled: true - health: - redis: - enabled: false - db: - enabled: false - endpoints: - web: - exposure: - include: '*' - exclude: env,auditevents,beans,conditions,info - base-path: /monitor +logging: + config: ./config/log4j2.xml + + diff --git a/config/avro/clickhouse/active_defence_event_log.avsc b/config/avro/clickhouse/active_defence_event_log.avsc deleted file mode 100644 index 56493301..00000000 --- a/config/avro/clickhouse/active_defence_event_log.avsc +++ /dev/null @@ -1,133 +0,0 @@ -{ - "type": "record", - "name": "active_defence_event_log", - "namespace": "tsg_galaxy_v3", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"schema_query\":{\"dimensions\":[\"common_policy_id\",\"ad_target_ip\",\"ad_cc_target_url\"],\"metrics\":[\"ad_target_ip\",\"ad_sent_byte_num\",\"ad_sent_pkt_num\",\"ad_cc_initiate_connection_num\",\"ad_cc_established_connection_num\",\"ad_cc_rejected_connection_num\"],\"filters\":[\"common_policy_id\",\"ad_target_ip\",\"ad_target_port\",\"ad_protocol\",\"common_address_type\",\"ad_sent_byte_num\",\"ad_sent_pkt_num\",\"ad_cc_initiate_connection_num\",\"ad_cc_established_connection_num\",\"ad_cc_rejected_connection_num\"]},\"schema_type\":{\"REFLECTION\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_address_type\",\"common_device_id\",\"common_link_id\",\"common_entrance_id\",\"common_user_region\",\"ad_method\",\"ad_protocol\",\"ad_target_ip\",\"ad_target_port\",\"ad_target_ip_location\",\"ad_target_ip_asn\",\"ad_reflector_profile_id\",\"ad_sent_pkt_num\",\"ad_sent_byte_num\",\"ad_generate_time\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"ad_target_ip\",\"ad_target_port\",\"ad_reflector_profile_id\",\"ad_sent_pkt_num\",\"ad_sent_byte_num\"]},\"FLOOD\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_address_type\",\"common_device_id\",\"common_link_id\",\"common_entrance_id\",\"common_user_region\",\"ad_method\",\"ad_protocol\",\"ad_target_ip\",\"ad_target_port\",\"ad_target_ip_location\",\"ad_target_ip_asn\",\"ad_claimed_src_ip_profile_id\",\"ad_sent_pkt_num\",\"ad_sent_byte_num\",\"ad_generate_time\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"ad_target_ip\",\"ad_target_port\",\"ad_claimed_src_ip_profile_id\",\"ad_protocol\"]},\"CC\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_address_type\",\"common_device_id\",\"common_link_id\",\"common_entrance_id\",\"common_user_region\",\"ad_method\",\"ad_protocol\",\"ad_cc_target_url\",\"ad_claimed_src_ip_profile_id\",\"ad_cc_initiate_connection_num\",\"ad_cc_established_connection_num\",\"ad_cc_rejected_connection_num\",\"ad_generate_time\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"ad_cc_target_url\",\"ad_claimed_src_ip_profile_id\",\"ad_protocol\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"ad_target_ip\",\"ad_target_port\",\"ad_cc_target_url\"]}", - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}" - }, - { - "name": "common_log_id", - "label": "Log ID", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_address_type", - "label": "Address Type", - "type": "int", - "doc": "{\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_device_id", - "label": "Device ID", - "type": "string" - }, - { - "name": "common_link_id", - "label": "Link ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_user_region", - "label": "User Region", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "ad_target_ip", - "label": "Target IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_country,geo_asn\",\"appendTo\":\"ad_target_ip_location,ad_target_ip_asn\"}}" - }, - { - "name": "ad_target_port", - "label": "Target Port", - "type": "int" - }, - { - "name": "ad_cc_target_url", - "label": "Target URL", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "ad_target_ip_location", - "label": "Target Location", - "type": "string" - }, - { - "name": "ad_target_ip_asn", - "label": "Target ASN", - "type": "string" - }, - { - "name": "ad_protocol", - "label": "Protocol", - "type": "string" - }, - { - "name": "ad_method", - "label": "Method", - "type": "string" - }, - { - "name": "ad_claimed_src_ip_profile_id", - "label": "Claimed Profile ID", - "type": "int" - }, - { - "name": "ad_reflector_profile_id", - "label": "Reflector Profile ID", - "type": "int" - }, - { - "name": "ad_sent_pkt_num", - "label": "Packets Sent", - "type": "int" - }, - { - "name": "ad_sent_byte_num", - "label": "Bytes Sent", - "type": "int" - }, - { - "name": "ad_cc_initiate_connection_num", - "label": "Initiate Numbers", - "type": "int" - }, - { - "name": "ad_cc_established_connection_num", - "label": "Established Numbers", - "type": "int" - }, - { - "name": "ad_cc_rejected_connection_num", - "label": "Rejected Numbers", - "type": "int" - }, - { - "name": "ad_generate_time", - "label": "Generate Time", - "type": "int", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/columns_cluster.avsc b/config/avro/clickhouse/columns_cluster.avsc deleted file mode 100644 index d190d3c0..00000000 --- a/config/avro/clickhouse/columns_cluster.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace": "system", - "type": "record", - "name": "columns_cluster", - "fields": [ - { - "name": "database", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/connection_record_log.avsc b/config/avro/clickhouse/connection_record_log.avsc deleted file mode 100644 index 17bbdd0f..00000000 --- a/config/avro/clickhouse/connection_record_log.avsc +++ /dev/null @@ -1,981 +0,0 @@ -{ - "type": "record", - "name": "connection_record_log", - "namespace": "tsg_galaxy_v3", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"index_table\":\"connection_record_log_common_client_ip,connection_record_log_common_server_ip,connection_record_log_common_subscriber_id,connection_record_log_http_domain\",\"functions\":{\"aggregation\":[{\"name\":\"COUNT\",\"function\":\"count(expr)\"},{\"name\":\"COUNT_DISTINCT\",\"function\":\"count(distinct expr)\"},{\"name\":\"AVG\",\"function\":\"avg(expr)\"},{\"name\":\"SUM\",\"function\":\"sum(expr)\"},{\"name\":\"MAX\",\"function\":\"max(expr)\"},{\"name\":\"MIN\",\"function\":\"min(expr)\"}],\"operator\":[{\"name\":\"=\",\"function\":\"expr = value\"},{\"name\":\"!=\",\"function\":\"expr != value\"},{\"name\":\">\",\"function\":\"expr > value\"},{\"name\":\"<\",\"function\":\"expr < value\"},{\"name\":\">=\",\"function\":\"expr >= value\"},{\"name\":\"<=\",\"function\":\"expr <= value\"},{\"name\":\"in\",\"function\":\"expr in (values)\"},{\"name\":\"not in\",\"function\":\"expr not in (values)\"},{\"name\":\"like\",\"function\":\"expr like value\"},{\"name\":\"not like\",\"function\":\"expr not like value\"},{\"name\":\"not empty\",\"function\":\"notEmpty(expr)\"},{\"name\":\"empty\",\"function\":\"empty(expr)\"}]},\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_sled_ip\",\"common_device_id\",\"common_client_location\",\"common_server_location\",\"common_subscriber_id\",\"common_client_port\",\"common_server_port\",\"common_schema_type\",\"common_l4_protocol\",\"common_l7_protocol\",\"common_data_center\",\"common_client_asn\",\"common_server_asn\",\"common_start_time\",\"common_end_time\",\"http_host\",\"http_domain\",\"http_url\",\"ssl_sni\",\"ssl_ja3_hash\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_subscriber_id\",\"common_sled_ip\",\"common_device_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_sessions\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"http_host\",\"http_domain\",\"http_url\",\"ssl_sni\",\"ssl_ja3_hash\"],\"filters\":[\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_client_port\",\"common_server_port\",\"common_client_location\",\"common_server_location\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_l4_protocol\",\"common_l7_protocol\",\"common_stream_dir\",\"common_direction\",\"common_data_center\",\"common_sled_ip\",\"common_device_id\",\"common_schema_type\",\"common_client_asn\",\"common_server_asn\",\"common_start_time\",\"common_end_time\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"http_host\",\"http_domain\",\"http_url\",\"ssl_sni\",\"ssl_ja3_hash\"],\"references\":{\"aggregation\":[{\"type\":\"int\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"long\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"float\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"double\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"string\",\"functions\":\"COUNT,COUNT_DISTINCT\"},{\"type\":\"date\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"},{\"type\":\"timestamp\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"}],\"operator\":[{\"type\":\"int\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"long\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"float\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"double\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"string\",\"functions\":\"=,!=,in,not in,like,not like,not empty,empty\"},{\"type\":\"date\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"timestamp\",\"functions\":\"=,!=,>,<,>=,<=\"}]},\"details\":{\"general\":[\"common_recv_time\",\"common_log_id\",\"common_stream_trace_id\",\"common_direction\",\"common_stream_dir\",\"common_start_time\",\"common_end_time\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"common_processing_time\"],\"action\":[\"common_action\",\"common_sub_action\",\"common_policy_id\",\"common_user_tags\",\"common_user_region\"],\"source\":[\"common_client_ip\",\"common_internal_ip\",\"common_client_port\",\"common_client_location\",\"common_client_asn\",\"common_entrance_id\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_data_center\",\"common_sled_ip\",\"common_tcp_client_isn\",\"common_subscriber_id\"],\"destination\":[\"common_server_ip\",\"common_external_ip\",\"common_server_port\",\"common_server_location\",\"common_server_asn\",\"common_tcp_server_isn\"],\"application\":[\"common_app_id\",\"common_app_label\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_protocol_label\",\"common_service\",\"common_l4_protocol\"],\"data\":[\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_first_ttl\"]}},\"schema_type\":{\"BASE\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\"]},\"HTTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"http_url\",\"http_host\",\"http_domain\",\"http_request_line\",\"http_response_line\",\"http_request_header\",\"http_response_header\",\"http_request_content\",\"http_response_content\",\"http_request_body\",\"http_response_body\",\"http_request_body_key\",\"http_response_body_key\",\"http_proxy_flag\",\"http_sequence\",\"http_snapshot\",\"http_cookie\",\"http_referer\",\"http_user_agent\",\"http_content_length\",\"http_content_type\",\"http_set_cookie\",\"http_version\",\"http_response_lantency_ms\",\"http_session_duration_ms\",\"http_action_file_size\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"http_url\",\"common_server_port\"]},\"MAIL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"mail_protocol_type\",\"mail_account\",\"mail_from_cmd\",\"mail_to_cmd\",\"mail_from\",\"mail_to\",\"mail_cc\",\"mail_bcc\",\"mail_subject\",\"mail_subject_charset\",\"mail_content\",\"mail_content_charset\",\"mail_attachment_name\",\"mail_attachment_name_charset\",\"mail_attachment_content\",\"mail_eml_file\",\"mail_snapshot\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"mail_from\",\"mail_to\",\"mail_subject\"]},\"DNS\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"dns_message_id\",\"dns_qr\",\"dns_opcode\",\"dns_aa\",\"dns_tc\",\"dns_rd\",\"dns_ra\",\"dns_rcode\",\"dns_qdcount\",\"dns_ancount\",\"dns_nscount\",\"dns_arcount\",\"dns_qname\",\"dns_qtype\",\"dns_qclass\",\"dns_cname\",\"dns_sub\",\"dns_rr\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_client_ip\",\"dns_qr\",\"dns_qname\",\"dns_qtype\"]},\"SSL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"ssl_sni\",\"ssl_san\",\"ssl_cn\",\"ssl_pinningst\",\"ssl_intercept_state\",\"ssl_server_side_latency\",\"ssl_client_side_latency\",\"ssl_server_side_version\",\"ssl_client_side_version\",\"ssl_cert_verify\",\"ssl_error\",\"ssl_con_latency_ms\",\"ssl_ja3_fingerprint\",\"ssl_ja3_hash\",\"ssl_cert_issuer\",\"ssl_cert_subject\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ssl_sni\",\"common_server_ip\",\"common_server_port\"]},\"QUIC\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"quic_version\",\"quic_sni\",\"quic_user_agent\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"quic_sni\",\"common_server_ip\",\"common_server_port\"]},\"FTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"ftp_account\",\"ftp_url\",\"ftp_content\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ftp_url\",\"common_server_ip\",\"common_server_port\"]},\"BGP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"bgp_type\",\"bgp_as_num\",\"bgp_route\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"bgp_type\",\"bgp_as_num\",\"common_server_ip\",\"common_server_port\"]},\"SIP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"sip_call_id\",\"sip_from\",\"sip_to\",\"sip_user_agent\",\"sip_server\",\"sip_from_sdp_connect_ip\",\"sip_from_sdp_media_port\",\"sip_from_sdp_media_type\",\"sip_from_sdp_content\",\"sip_to_sdp_connect_ip\",\"sip_to_sdp_media_port\",\"sip_to_sdp_media_type\",\"sip_to_sdp_content\",\"sip_duration\",\"sip_bye\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"sip_from\",\"sip_to\",\"sip_call_id\",\"common_server_ip\",\"common_server_port\"]},\"RTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"rtp_payload_type_c2s\",\"rtp_payload_type_s2c\",\"rtp_pcap_dir_c2s\",\"rtp_pcap_dir_s2c\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"rtp_pcap_dir_c2s\",\"rtp_pcap_dir_s2c\",\"rtp_from_direction\",\"rtp_from_direction\"]},\"APP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"app_extra_info\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_app_id\",\"common_app_label\",\"app_extra_info\",\"common_server_ip\",\"common_server_port\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"common_schema_type\"]}", - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_log_id", - "label": "Log ID", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_subscriber_id", - "label": "Subscriber ID", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "label": "Client IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_asn,radius_match\",\"appendTo\":\"common_client_asn,common_subscriber_id\"}}" - }, - { - "name": "common_internal_ip", - "label": "Internal IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"if\",\"param\":\"$.common_direction=69,$.common_client_ip,$.common_server_ip\"}}" - }, - { - "name": "common_client_port", - "label": "Client Port", - "type": "int", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_l4_protocol", - "label": "L4 Protocol", - "type": "string" - }, - { - "name": "common_address_type", - "label": "Address Type", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "label": "Server IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_asn\",\"appendTo\":\"common_server_asn\"}}" - }, - { - "name": "common_server_port", - "label": "Server Port", - "type": "int", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_external_ip", - "label": "External IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"if\",\"param\":\"$.common_direction=73,$.common_client_ip,$.common_server_ip\"}}" - }, - { - "name": "common_action", - "label": "Action", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"128\",\"value\":\"Allow\"}]}" - }, - { - "name": "common_direction", - "label": "Direction", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"69\",\"value\":\"outbound\"},{\"code\":\"73\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "label": "Sled IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "label": "Client Location", - "type": "string" - }, - { - "name": "common_client_asn", - "label": "Client ASN", - "type": "string" - }, - { - "name": "common_server_location", - "label": "Server Location", - "type": "string" - }, - { - "name": "common_server_asn", - "label": "Server ASN", - "type": "string" - }, - { - "name": "common_sessions", - "label": "Sessions", - "type": "long" - }, - { - "name": "common_c2s_pkt_num", - "label": "Packets Sent", - "type": "long" - }, - { - "name": "common_s2c_pkt_num", - "label": "Packets Received", - "type": "long" - }, - { - "name": "common_c2s_byte_num", - "label": "Bytes Sent", - "type": "long" - }, - { - "name": "common_s2c_byte_num", - "label": "Bytes Received", - "type": "long" - }, - { - "name": "common_service", - "label": "Service", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "label": "Schema Type", - "type": "string", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"BASE\",\"value\":\"BASE\"},{\"code\":\"MAIL\",\"value\":\"MAIL\"},{\"code\":\"DNS\",\"value\":\"DNS\"},{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"SSL\",\"value\":\"SSL\"},{\"code\":\"FTP\",\"value\":\"FTP\"},{\"code\":\"SIP\",\"value\":\"SIP\"},{\"code\":\"RTP\",\"value\":\"RTP\"},{\"code\":\"APP\",\"value\":\"APP\"}],\"allow_query\":\"true\"}" - }, - { - "name": "common_user_tags", - "label": "User Tags", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "label": "Sub Action", - "type": "string" - }, - { - "name": "common_user_region", - "label": "User Region", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "label": "Device ID", - "type": "string" - }, - { - "name": "common_link_id", - "label": "Link ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "label": "ISP", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_device_tag", - "label": "Device Tag", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_data_center", - "label": "Data Center", - "type": "string", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"Nur-sultan\",\"value\":\"Nur-sultan\"},{\"code\":\"Aktau\",\"value\":\"Aktau\"},{\"code\":\"Aktubinsk\",\"value\":\"Aktubinsk\"},{\"code\":\"Almaty\",\"value\":\"Almaty\"},{\"code\":\"Atyrau\",\"value\":\"Atyrau\"},{\"code\":\"Karaganda\",\"value\":\"Karaganda\"},{\"code\":\"Kokshetau\",\"value\":\"Kokshetau\"},{\"code\":\"Kostanay\",\"value\":\"Kostanay\"},{\"code\":\"Kyzylorda\",\"value\":\"Kyzylorda\"},{\"code\":\"Pavlodar\",\"value\":\"Pavlodar\"},{\"code\":\"Petropavl\",\"value\":\"Petropavl\"},{\"code\":\"Semey\",\"value\":\"Semey\"},{\"code\":\"Shymkent\",\"value\":\"Shymkent\"},{\"code\":\"Taldykurgan\",\"value\":\"Taldykurgan\"},{\"code\":\"Taraz\",\"value\":\"Taraz\"},{\"code\":\"Uralsk\",\"value\":\"Uralsk\"},{\"code\":\"Ust-Kamenogorsk\",\"value\":\"Ust-Kamenogorsk\"},{\"code\":\"Zhezkazgan\",\"value\":\"Zhezkazgan\"}],\"allow_query\":\"true\"}" - }, - { - "name": "common_encapsulation", - "label": "Encapsulation", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_label", - "label": "Application Label", - "type": "string" - }, - { - "name": "common_protocol_label", - "label": "Protocol Label", - "type": "string" - }, - { - "name": "common_app_id", - "label": "Application ID", - "type": "int" - }, - { - "name": "common_app_surrogate_id", - "label": "Surrogate ID", - "type": "int" - }, - { - "name": "common_l7_protocol", - "label": "L7 Protocol", - "type": "string" - }, - { - "name": "common_start_time", - "label": "Start Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_end_time", - "label": "End Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"get_value\",\"appendTo\":\"common_recv_time\"}}" - }, - { - "name": "common_establish_latency_ms", - "label": "Establish Latency(ms)", - "type": "int" - }, - { - "name": "common_con_duration_ms", - "label": "Duration(ms)", - "type": "int" - }, - { - "name": "common_stream_dir", - "label": "Stream Direction", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"allow_query\":\"true\",\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "label": "Address List", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_has_dup_traffic", - "label": "Duplication Traffic", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "label": "Stream Error", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_trace_id", - "label": "Session ID", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_ipfrag_num", - "label": "Fragmentation Packets(c2s)", - "type": "long" - }, - { - "name": "common_s2c_ipfrag_num", - "label": "Fragmentation Packets(s2c)", - "type": "long" - }, - { - "name": "common_c2s_tcp_lostlen", - "label": "Sequence Gap Loss(c2s)", - "type": "long" - }, - { - "name": "common_s2c_tcp_lostlen", - "label": "Sequence Gap Loss(s2c)", - "type": "long" - }, - { - "name": "common_c2s_tcp_unorder_num", - "label": "Unorder Packets(c2s)", - "type": "long" - }, - { - "name": "common_s2c_tcp_unorder_num", - "label": "Unorder Packets(s2c)", - "type": "long" - }, - { - "name": "common_tcp_client_isn", - "label": "TCP Client ISN", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_tcp_server_isn", - "label": "TCP Server ISN", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_first_ttl", - "label": "First TTL", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_processing_time", - "label": "Processing Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}", - "type": "long" - }, - { - "name": "http_url", - "label": "Http.URL", - "type": "string" - }, - { - "name": "http_host", - "label": "Http.Host", - "type": "string", - "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "http_domain", - "label": "Http.Domain", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "http_request_line", - "label": "Http.Request Line", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_response_line", - "label": "Http.Response Line", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_request_header", - "label": "Http.Request Headers", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_header", - "label": "Http.Response Headers", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_content", - "label": "Http.Request Content", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_content", - "label": "Http.Response Content", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_body", - "label": "Http.Request Body", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_response_body", - "label": "Http.Response Body", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_request_body_key", - "label": "Http.Request Body Key", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_response_body_key", - "label": "Http.Response Body Key", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_proxy_flag", - "label": "Http.Proxy Flag", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_sequence", - "label": "Http.Sequence", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_snapshot", - "label": "Http.Snapshot", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_cookie", - "label": "Http.Cookie", - "type": "string" - }, - { - "name": "http_referer", - "label": "Http.Referer", - "type": "string" - }, - { - "name": "http_user_agent", - "label": "Http.User Agent", - "type": "string" - }, - { - "name": "http_content_length", - "label": "Http.Content Length", - "type": "string" - }, - { - "name": "http_content_type", - "label": "Http.Content Type", - "type": "string" - }, - { - "name": "http_set_cookie", - "label": "Http.Set Cookie", - "type": "string" - }, - { - "name": "http_version", - "label": "Http.Version", - "type": "string" - }, - { - "name": "http_response_lantency_ms", - "label": "Http.Response Latency(ms)", - "type": "int" - }, - { - "name": "http_session_duration_ms", - "label": "Http.Session Duration(ms)", - "type": "int" - }, - { - "name": "http_action_file_size", - "label": "Http.Action File Size", - "type": "int" - }, - { - "name": "mail_protocol_type", - "label": "Mail.Protocol Type", - "type": "string" - }, - { - "name": "mail_account", - "label": "Mail.Account", - "type": "string" - }, - { - "name": "mail_from_cmd", - "label": "Mail.From CMD", - "type": "string" - }, - { - "name": "mail_to_cmd", - "label": "Mail.To CMD", - "type": "string" - }, - { - "name": "mail_from", - "label": "Mail.From", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"email\"}}" - }, - { - "name": "mail_to", - "label": "Mail.To", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"email\"}}" - }, - { - "name": "mail_cc", - "label": "Mail.CC", - "type": "string" - }, - { - "name": "mail_bcc", - "label": "Mail.BCC", - "type": "string" - }, - { - "name": "mail_subject", - "label": "Mail.Subject", - "type": "string", - "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"$.mail_subject_charset\"}}" - }, - { - "name": "mail_subject_charset", - "label": "Mail.Subject Charset", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_content", - "label": "Mail.Content", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_content_charset", - "label": "Mail.Content Charset", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_attachment_name", - "label": "Mail.Attachment", - "type": "string", - "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"$.mail_attachment_name_charset\"}}" - }, - { - "name": "mail_attachment_name_charset", - "label": "Mail.Attachment Charset", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_attachment_content", - "label": "Mail.Attachment Content", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_eml_file", - "label": "Mail.EML File", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "mail_snapshot", - "label": "Mail.Snapshot", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "dns_message_id", - "label": "Dns.Message ID", - "type": "int" - }, - { - "name": "dns_qr", - "label": "Dns.QR", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"REESPONSE\"}]}" - }, - { - "name": "dns_opcode", - "label": "Dns.OPCODE", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"IQUERY\"},{\"code\":\"2\",\"value\":\"STATUS\"},{\"code\":\"5\",\"value\":\"UPDATE\"}]}" - }, - { - "name": "dns_aa", - "label": "Dns.AA", - "type": "int" - }, - { - "name": "dns_tc", - "label": "Dns.TC", - "type": "int" - }, - { - "name": "dns_rd", - "label": "Dns.RD", - "type": "int" - }, - { - "name": "dns_ra", - "label": "Dns.RA", - "type": "int" - }, - { - "name": "dns_rcode", - "label": "Dns.RCODE", - "type": "int" - }, - { - "name": "dns_qdcount", - "label": "Dns.QDCOUNT", - "type": "int" - }, - { - "name": "dns_ancount", - "label": "Dns.ANCOUNT", - "type": "int" - }, - { - "name": "dns_nscount", - "label": "Dns.NSCOUNT", - "type": "int" - }, - { - "name": "dns_arcount", - "label": "Dns.ARCOUNT", - "type": "int" - }, - { - "name": "dns_qname", - "label": "Dns.QNAME", - "type": "string" - }, - { - "name": "dns_qtype", - "label": "Dns.QTYPE", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"A\"},{\"code\":\"2\",\"value\":\"NS\"},{\"code\":\"5\",\"value\":\"CNAME\"},{\"code\":\"6\",\"value\":\"SOA\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"12\",\"value\":\"PTR\"},{\"code\":\"13\",\"value\":\"HINFO\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"15\",\"value\":\"MX\"},{\"code\":\"28\",\"value\":\"AAAA\"}]}" - }, - { - "name": "dns_qclass", - "label": "Dns.QCLASS", - "type": "int" - }, - { - "name": "dns_cname", - "label": "Dns.CNAME", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "dns_sub", - "label": "Dns.SUB", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"DNS\"},{\"code\":\"2\",\"value\":\"DNSSEC\"}]}" - }, - { - "name": "dns_rr", - "label": "Dns.RR", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "ssl_version", - "label": "SSL.Version", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "ssl_sni", - "label": "SSL.SNI", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "ssl_san", - "label": "SSL.SAN", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "ssl_cn", - "label": "SSL.CN", - "type": "string" - }, - { - "name": "ssl_pinningst", - "label": "SSL.Pinning", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"Not Pinning\"},{\"code\":\"1\",\"value\":\"Pinning\"},{\"code\":\"2\",\"value\":\"Maybe Pinning\"}]}" - }, - { - "name": "ssl_intercept_state", - "label": "SSL.Intercept State", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"Passthrough\"},{\"code\":\"1\",\"value\":\"Intercept\"},{\"code\":\"2\",\"value\":\"Shutdown\"}]}" - }, - { - "name": "ssl_server_side_latency", - "label": "SSL.Server Side Latency(ms)", - "type": "int" - }, - { - "name": "ssl_client_side_latency", - "label": "SSL.Client Side Latency(ms)", - "type": "int" - }, - { - "name": "ssl_server_side_version", - "label": "SSL.Server Side Version", - "type": "string" - }, - { - "name": "ssl_client_side_version", - "label": "SSL.Client Side Version", - "type": "string" - }, - { - "name": "ssl_cert_verify", - "label": "SSL.Certificate Verify", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}]}" - }, - { - "name": "ssl_error", - "label": "SSL.Error", - "type": "string" - }, - { - "name": "ssl_con_latency_ms", - "label": "SSL.Connection Latency(ms)", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "ssl_ja3_fingerprint", - "label": "SSL.JA3", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "ssl_ja3_hash", - "label": "SSL.JA3 hash", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "ssl_cert_issuer", - "label": "SSL.Issuer", - "type": "string" - }, - { - "name": "ssl_cert_subject", - "label": "SSL.Subject", - "type": "string" - }, - { - "name": "quic_version", - "label": "Quic.Version", - "type": "string" - }, - { - "name": "quic_sni", - "label": "Quic.SNI", - "type": "string", - "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "quic_user_agent", - "label": "Quic.User Agent", - "type": "string" - }, - { - "name": "ftp_account", - "label": "Ftp.Account", - "type": "string" - }, - { - "name": "ftp_url", - "label": "Ftp.URL", - "type": "string" - }, - { - "name": "ftp_content", - "label": "Ftp.Content", - "type": "string" - }, - { - "name": "bgp_type", - "label": "BGP.Type", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "bgp_as_num", - "label": "BGP.AS Number", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "bgp_route", - "label": "BGP.Route", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_calling_account", - "label": "Voip.Calling Account", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_called_account", - "label": "Voip.Called Account", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_calling_number", - "label": "Voip.Calling Number", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_called_number", - "label": "Voip.Called Number", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "streaming_media_url", - "label": "Streaming.Media URL", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "streaming_media_protocol", - "label": "Streaming.Media Protocol", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "app_extra_info", - "label": "APP.Extra Info", - "type": "string" - }, - { - "name": "sip_call_id", - "label": "SIP.Call-ID", - "type": "string" - }, - { - "name": "sip_from", - "label": "SIP.From", - "type": "string" - }, - { - "name": "sip_to", - "label": "SIP.To", - "type": "string" - }, - { - "name": "sip_user_agent", - "label": "SIP.User-Agent", - "type": "string" - }, - { - "name": "sip_server", - "label": "SIP.Server", - "type": "string" - }, - { - "name": "sip_from_sdp_connect_ip", - "label": "SIP.From Connect IP", - "type": "string" - }, - { - "name": "sip_from_sdp_media_port", - "label": "SIP.From Media Port", - "type": "int" - }, - { - "name": "sip_from_sdp_media_type", - "label": "SIP.From Media Type", - "type": "string" - }, - { - "name": "sip_from_sdp_content", - "label": "SIP.From SDP Content", - "type": "string" - }, - { - "name": "sip_to_sdp_connect_ip", - "label": "SIP.To Connect IP", - "type": "string" - }, - { - "name": "sip_to_sdp_media_port", - "label": "SIP.To Media Port", - "type": "int" - }, - { - "name": "sip_to_sdp_media_type", - "label": "SIP.To Media Type", - "type": "string" - }, - { - "name": "sip_to_sdp_content", - "label": "SIP.To SDP Content", - "type": "string" - }, - { - "name": "sip_duration", - "label": "SIP.Duration", - "type": "int" - }, - { - "name": "sip_bye", - "label": "SIP.Bye", - "type": "string" - }, - { - "name": "rtp_payload_type_c2s", - "label": "RTP.Payload(c2s)", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"PCMU\"},{\"code\":\"1\",\"value\":\"1016\"},{\"code\":\"2\",\"value\":\"G721\"},{\"code\":\"3\",\"value\":\"GSM\"},{\"code\":\"4\",\"value\":\"G723\"},{\"code\":\"5\",\"value\":\"DVI4_8000\"},{\"code\":\"6\",\"value\":\"DVI4_16000\"},{\"code\":\"7\",\"value\":\"LPC\"},{\"code\":\"8\",\"value\":\"PCMA\"},{\"code\":\"9\",\"value\":\"G722\"},{\"code\":\"10\",\"value\":\"L16_STEREO\"},{\"code\":\"11\",\"value\":\"L16_MONO\"},{\"code\":\"12\",\"value\":\"QCELP\"},{\"code\":\"13\",\"value\":\"CN\"},{\"code\":\"14\",\"value\":\"MPA\"},{\"code\":\"15\",\"value\":\"G728\"},{\"code\":\"16\",\"value\":\"DVI4_11025\"},{\"code\":\"17\",\"value\":\"DVI4_22050\"},{\"code\":\"18\",\"value\":\"G729\"},{\"code\":\"19\",\"value\":\"CN_OLD\"},{\"code\":\"25\",\"value\":\"CELB\"},{\"code\":\"26\",\"value\":\"JPEG\"},{\"code\":\"28\",\"value\":\"NV\"},{\"code\":\"31\",\"value\":\"H261\"},{\"code\":\"32\",\"value\":\"MPV\"},{\"code\":\"33\",\"value\":\"MP2T\"},{\"code\":\"34\",\"value\":\"H263\"}]}", - "type": "int" - }, - { - "name": "rtp_payload_type_s2c", - "label": "RTP.Payload(s2c)", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"PCMU\"},{\"code\":\"1\",\"value\":\"1016\"},{\"code\":\"2\",\"value\":\"G721\"},{\"code\":\"3\",\"value\":\"GSM\"},{\"code\":\"4\",\"value\":\"G723\"},{\"code\":\"5\",\"value\":\"DVI4_8000\"},{\"code\":\"6\",\"value\":\"DVI4_16000\"},{\"code\":\"7\",\"value\":\"LPC\"},{\"code\":\"8\",\"value\":\"PCMA\"},{\"code\":\"9\",\"value\":\"G722\"},{\"code\":\"10\",\"value\":\"L16_STEREO\"},{\"code\":\"11\",\"value\":\"L16_MONO\"},{\"code\":\"12\",\"value\":\"QCELP\"},{\"code\":\"13\",\"value\":\"CN\"},{\"code\":\"14\",\"value\":\"MPA\"},{\"code\":\"15\",\"value\":\"G728\"},{\"code\":\"16\",\"value\":\"DVI4_11025\"},{\"code\":\"17\",\"value\":\"DVI4_22050\"},{\"code\":\"18\",\"value\":\"G729\"},{\"code\":\"19\",\"value\":\"CN_OLD\"},{\"code\":\"25\",\"value\":\"CELB\"},{\"code\":\"26\",\"value\":\"JPEG\"},{\"code\":\"28\",\"value\":\"NV\"},{\"code\":\"31\",\"value\":\"H261\"},{\"code\":\"32\",\"value\":\"MPV\"},{\"code\":\"33\",\"value\":\"MP2T\"},{\"code\":\"34\",\"value\":\"H263\"}]}", - "type": "int" - }, - { - "name": "rtp_pcap_dir_c2s", - "label": "RTP.PCAP(c2s)", - "doc": "{\"constraints\":{\"type\":\"file\"}}", - "type": "string" - }, - { - "name": "rtp_pcap_dir_s2c", - "label": "RTP.PCAP(s2c)", - "doc": "{\"constraints\":{\"type\":\"file\"}}", - "type": "string" - }, - { - "name": "rtp_from_direction", - "label": "Direction", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"unknown\"},{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"}]}", - "type": "int" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/connection_record_log_common_client_ip.avsc b/config/avro/clickhouse/connection_record_log_common_client_ip.avsc deleted file mode 100644 index 98215cee..00000000 --- a/config/avro/clickhouse/connection_record_log_common_client_ip.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "connection_record_log_common_client_ip", - "doc": "{\"primary_key\":\"common_log_id\",\"index_key\":\"common_client_ip\"}", - "fields": [ - { - "name": "common_log_id", - "type": "long" - }, - { - "name": "common_recv_time", - "type": "long" - }, - { - "name": "common_policy_id", - "type": "long" - }, - { - "name": "common_action", - "type": "int" - }, - { - "name": "common_server_ip", - "type": "string" - }, - { - "name": "common_client_ip", - "type": "string" - }, - { - "name": "common_sled_ip", - "type": "string" - }, - { - "name": "common_entrance_id", - "type": "int" - }, - { - "name": "common_subscriber_id", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "type": "long" - }, - { - "name": "http_domain", - "type": "string" - }, - { - "name": "ssl_sni", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/connection_record_log_common_server_ip.avsc b/config/avro/clickhouse/connection_record_log_common_server_ip.avsc deleted file mode 100644 index b47449cd..00000000 --- a/config/avro/clickhouse/connection_record_log_common_server_ip.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "connection_record_log_common_server_ip", - "doc": "{\"primary_key\":\"common_log_id\",\"index_key\":\"common_server_ip\"}", - "fields": [ - { - "name": "common_log_id", - "type": "long" - }, - { - "name": "common_recv_time", - "type": "long" - }, - { - "name": "common_policy_id", - "type": "long" - }, - { - "name": "common_action", - "type": "int" - }, - { - "name": "common_server_ip", - "type": "string" - }, - { - "name": "common_client_ip", - "type": "string" - }, - { - "name": "common_sled_ip", - "type": "string" - }, - { - "name": "common_entrance_id", - "type": "int" - }, - { - "name": "common_subscriber_id", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "type": "long" - }, - { - "name": "http_domain", - "type": "string" - }, - { - "name": "ssl_sni", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/connection_record_log_common_subscriber_id.avsc b/config/avro/clickhouse/connection_record_log_common_subscriber_id.avsc deleted file mode 100644 index 358fe9e0..00000000 --- a/config/avro/clickhouse/connection_record_log_common_subscriber_id.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "connection_record_log_common_subscriber_id", - "doc": "{\"primary_key\":\"common_log_id\",\"index_key\":\"common_subscriber_id\"}", - "fields": [ - { - "name": "common_log_id", - "type": "long" - }, - { - "name": "common_recv_time", - "type": "long" - }, - { - "name": "common_policy_id", - "type": "long" - }, - { - "name": "common_action", - "type": "int" - }, - { - "name": "common_server_ip", - "type": "string" - }, - { - "name": "common_client_ip", - "type": "string" - }, - { - "name": "common_sled_ip", - "type": "string" - }, - { - "name": "common_entrance_id", - "type": "int" - }, - { - "name": "common_subscriber_id", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "type": "long" - }, - { - "name": "http_domain", - "type": "string" - }, - { - "name": "ssl_sni", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/connection_record_log_http_domain.avsc b/config/avro/clickhouse/connection_record_log_http_domain.avsc deleted file mode 100644 index 0ca111cb..00000000 --- a/config/avro/clickhouse/connection_record_log_http_domain.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "connection_record_log_http_domain", - "doc": "{\"primary_key\":\"common_log_id\",\"index_key\":\"http_domain\"}", - "fields": [ - { - "name": "common_log_id", - "type": "long" - }, - { - "name": "common_recv_time", - "type": "long" - }, - { - "name": "common_policy_id", - "type": "long" - }, - { - "name": "common_action", - "type": "int" - }, - { - "name": "common_server_ip", - "type": "string" - }, - { - "name": "common_client_ip", - "type": "string" - }, - { - "name": "common_sled_ip", - "type": "string" - }, - { - "name": "common_entrance_id", - "type": "int" - }, - { - "name": "common_subscriber_id", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "type": "long" - }, - { - "name": "http_domain", - "type": "string" - }, - { - "name": "ssl_sni", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/disks_cluster.avsc b/config/avro/clickhouse/disks_cluster.avsc deleted file mode 100644 index 70777c69..00000000 --- a/config/avro/clickhouse/disks_cluster.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace": "system", - "type": "record", - "name": "disks_cluster", - "fields": [ - { - "name": "name", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/parts_cluster.avsc b/config/avro/clickhouse/parts_cluster.avsc deleted file mode 100644 index c311abf3..00000000 --- a/config/avro/clickhouse/parts_cluster.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace": "system", - "type": "record", - "name": "parts_cluster", - "fields": [ - { - "name": "name", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/processes.avsc b/config/avro/clickhouse/processes.avsc deleted file mode 100644 index 75d74a98..00000000 --- a/config/avro/clickhouse/processes.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace": "system", - "type": "record", - "name": "processes", - "fields": [ - { - "name": "query_id", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/proxy_event_log.avsc b/config/avro/clickhouse/proxy_event_log.avsc deleted file mode 100644 index 98805398..00000000 --- a/config/avro/clickhouse/proxy_event_log.avsc +++ /dev/null @@ -1,668 +0,0 @@ -{ - "type": "record", - "name": "proxy_event_log", - "namespace": "tsg_galaxy_v3", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"functions\":{\"aggregation\":[{\"name\":\"COUNT\",\"function\":\"count(expr)\"},{\"name\":\"COUNT_DISTINCT\",\"function\":\"count(distinct expr)\"},{\"name\":\"AVG\",\"function\":\"avg(expr)\"},{\"name\":\"SUM\",\"function\":\"sum(expr)\"},{\"name\":\"MAX\",\"function\":\"max(expr)\"},{\"name\":\"MIN\",\"function\":\"min(expr)\"}],\"operator\":[{\"name\":\"=\",\"function\":\"expr = value\"},{\"name\":\"!=\",\"function\":\"expr != value\"},{\"name\":\">\",\"function\":\"expr > value\"},{\"name\":\"<\",\"function\":\"expr < value\"},{\"name\":\">=\",\"function\":\"expr >= value\"},{\"name\":\"<=\",\"function\":\"expr <= value\"},{\"name\":\"in\",\"function\":\"expr in (values)\"},{\"name\":\"not in\",\"function\":\"expr not in (values)\"},{\"name\":\"like\",\"function\":\"expr like value\"},{\"name\":\"not like\",\"function\":\"expr not like value\"},{\"name\":\"not empty\",\"function\":\"notEmpty(expr)\"},{\"name\":\"empty\",\"function\":\"empty(expr)\"}]},\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_policy_id\",\"common_sub_action\",\"common_sled_ip\",\"common_device_id\",\"common_client_location\",\"common_server_location\",\"common_subscriber_id\",\"common_client_port\",\"common_server_port\",\"common_schema_type\",\"common_data_center\",\"common_client_asn\",\"common_server_asn\",\"http_host\",\"http_domain\",\"http_url\",\"doh_host\",\"doh_qname\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_subscriber_id\",\"common_sled_ip\",\"common_device_id\",\"common_sessions\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"http_host\",\"http_domain\",\"http_url\",\"doh_host\",\"doh_qname\"],\"filters\":[\"common_policy_id\",\"common_sub_action\",\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_client_port\",\"common_server_port\",\"common_client_location\",\"common_server_location\",\"common_subscriber_id\",\"common_l4_protocol\",\"common_data_center\",\"common_sled_ip\",\"common_device_id\",\"common_client_asn\",\"common_server_asn\",\"common_direction\",\"common_schema_type\",\"http_host\",\"http_domain\",\"http_url\",\"http_content_type\",\"doh_host\",\"doh_qname\"],\"references\":{\"aggregation\":[{\"type\":\"int\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"long\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"float\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"double\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"string\",\"functions\":\"COUNT,COUNT_DISTINCT\"},{\"type\":\"date\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"},{\"type\":\"timestamp\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"}],\"operator\":[{\"type\":\"int\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"long\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"float\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"double\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"string\",\"functions\":\"=,!=,in,not in,like,not like,not empty,empty\"},{\"type\":\"date\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"timestamp\",\"functions\":\"=,!=,>,<,>=,<=\"}]},\"details\":{\"general\":[\"common_recv_time\",\"common_log_id\",\"common_stream_trace_id\",\"common_direction\",\"common_stream_dir\",\"common_start_time\",\"common_end_time\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"common_processing_time\"],\"action\":[\"common_action\",\"common_sub_action\",\"common_policy_id\",\"common_user_tags\",\"common_user_region\"],\"source\":[\"common_client_ip\",\"common_internal_ip\",\"common_client_port\",\"common_client_location\",\"common_client_asn\",\"common_entrance_id\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_data_center\",\"common_sled_ip\",\"common_tcp_client_isn\",\"common_subscriber_id\"],\"destination\":[\"common_server_ip\",\"common_external_ip\",\"common_server_port\",\"common_server_location\",\"common_server_asn\",\"common_tcp_server_isn\"],\"application\":[\"common_app_id\",\"common_app_label\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_protocol_label\",\"common_service\",\"common_l4_protocol\"],\"data\":[\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_first_ttl\"]}},\"schema_type\":{\"HTTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"http_url\",\"http_host\",\"http_domain\",\"http_request_line\",\"http_response_line\",\"http_request_header\",\"http_response_header\",\"http_request_content\",\"http_response_content\",\"http_request_body\",\"http_response_body\",\"http_request_body_key\",\"http_response_body_key\",\"http_proxy_flag\",\"http_sequence\",\"http_snapshot\",\"http_cookie\",\"http_referer\",\"http_user_agent\",\"http_content_length\",\"http_content_type\",\"http_set_cookie\",\"http_version\",\"http_response_lantency_ms\",\"http_session_duration_ms\",\"http_action_file_size\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"http_url\",\"common_sub_action\"]},\"DoH\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"doh_url\",\"doh_host\",\"doh_request_line\",\"doh_response_line\",\"doh_cookie\",\"doh_referer\",\"doh_user_agent\",\"doh_content_length\",\"doh_content_type\",\"doh_set_cookie\",\"doh_version\",\"doh_message_id\",\"doh_qr\",\"doh_opcode\",\"doh_aa\",\"doh_tc\",\"doh_rd\",\"doh_ra\",\"doh_rcode\",\"doh_qdcount\",\"doh_ancount\",\"doh_nscount\",\"doh_arcount\",\"doh_qname\",\"doh_qtype\",\"doh_qclass\",\"doh_cname\",\"doh_sub\",\"doh_rr\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_client_ip\",\"doh_url\",\"doh_qname\",\"common_server_port\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"common_sub_action\",\"common_schema_type\"]}", - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_log_id", - "label": "Log ID", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_subscriber_id", - "label": "Subscriber ID", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "label": "Client IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_asn,radius_match\",\"appendTo\":\"common_client_asn,common_subscriber_id\"}}" - }, - { - "name": "common_internal_ip", - "label": "Internal IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"if\",\"param\":\"$.common_direction=69,$.common_client_ip,$.common_server_ip\"}}" - }, - { - "name": "common_client_port", - "label": "Client Port", - "type": "int", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_l4_protocol", - "label": "L4 Protocol", - "type": "string" - }, - { - "name": "common_address_type", - "label": "Address Type", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "label": "Server IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_asn\",\"appendTo\":\"common_server_asn\"}}" - }, - { - "name": "common_server_port", - "label": "Server Port", - "type": "int", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_external_ip", - "label": "External IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"if\",\"param\":\"$.common_direction=73,$.common_client_ip,$.common_server_ip\"}}" - }, - { - "name": "common_action", - "label": "Action", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"48\",\"value\":\"Manipulation\"},{\"code\":\"128\",\"value\":\"Allow\"}]}" - }, - { - "name": "common_direction", - "label": "Direction", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"69\",\"value\":\"outbound\"},{\"code\":\"73\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "label": "Sled IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "label": "Client Location", - "type": "string" - }, - { - "name": "common_client_asn", - "label": "Client ASN", - "type": "string" - }, - { - "name": "common_server_location", - "label": "Server Location", - "type": "string" - }, - { - "name": "common_server_asn", - "label": "Server ASN", - "type": "string" - }, - { - "name": "common_sessions", - "label": "Sessions", - "type": "long", - "doc": "{\"format\":{\"functions\":\"set_value\",\"param\":\"1\"}}" - }, - { - "name": "common_c2s_pkt_num", - "label": "Packets Sent", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_pkt_num", - "label": "Packets Received", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_byte_num", - "label": "Bytes Sent", - "type": "long" - }, - { - "name": "common_s2c_byte_num", - "label": "Bytes Received", - "type": "long" - }, - { - "name": "common_service", - "label": "Service", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "label": "Schema Type", - "type": "string", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"allow_query\":\"true\",\"data\":[{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"DoH\",\"value\":\"DoH\"}]}" - }, - { - "name": "common_user_tags", - "label": "User Tags", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "label": "Sub Action", - "type": "string", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"allow\",\"value\":\"allow\"},{\"code\":\"deny\",\"value\":\"deny\"},{\"code\":\"monitor\",\"value\":\"monitor\"},{\"code\":\"replace\",\"value\":\"replace\"},{\"code\":\"redirect\",\"value\":\"redirect\"},{\"code\":\"insert\",\"value\":\"insert\"},{\"code\":\"hijack\",\"value\":\"hijack\"}],\"allow_query\":\"true\"}" - }, - { - "name": "common_user_region", - "label": "User Region", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "label": "Device ID", - "type": "string" - }, - { - "name": "common_link_id", - "label": "Link ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "label": "ISP", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_device_tag", - "label": "Device Tag", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_data_center", - "label": "Data Center", - "type": "string", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"Nur-sultan\",\"value\":\"Nur-sultan\"},{\"code\":\"Aktau\",\"value\":\"Aktau\"},{\"code\":\"Aktubinsk\",\"value\":\"Aktubinsk\"},{\"code\":\"Almaty\",\"value\":\"Almaty\"},{\"code\":\"Atyrau\",\"value\":\"Atyrau\"},{\"code\":\"Karaganda\",\"value\":\"Karaganda\"},{\"code\":\"Kokshetau\",\"value\":\"Kokshetau\"},{\"code\":\"Kostanay\",\"value\":\"Kostanay\"},{\"code\":\"Kyzylorda\",\"value\":\"Kyzylorda\"},{\"code\":\"Pavlodar\",\"value\":\"Pavlodar\"},{\"code\":\"Petropavl\",\"value\":\"Petropavl\"},{\"code\":\"Semey\",\"value\":\"Semey\"},{\"code\":\"Shymkent\",\"value\":\"Shymkent\"},{\"code\":\"Taldykurgan\",\"value\":\"Taldykurgan\"},{\"code\":\"Taraz\",\"value\":\"Taraz\"},{\"code\":\"Uralsk\",\"value\":\"Uralsk\"},{\"code\":\"Ust-Kamenogorsk\",\"value\":\"Ust-Kamenogorsk\"},{\"code\":\"Zhezkazgan\",\"value\":\"Zhezkazgan\"}],\"allow_query\":\"true\"}" - }, - { - "name": "common_encapsulation", - "label": "Encapsulation", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_label", - "label": "Application Label", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_protocol_label", - "label": "Protocol Label", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_id", - "label": "Application ID", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_surrogate_id", - "label": "Surrogate ID", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_l7_protocol", - "label": "L7 Protocol", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_start_time", - "label": "Start Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_end_time", - "label": "End Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"get_value\",\"appendTo\":\"common_recv_time\"}}" - }, - { - "name": "common_establish_latency_ms", - "label": "Establish Latency(ms)", - "type": "int" - }, - { - "name": "common_con_duration_ms", - "label": "Duration(ms)", - "type": "int" - }, - { - "name": "common_stream_dir", - "label": "Stream Direction", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "label": "Address List", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_has_dup_traffic", - "label": "Duplication Traffic", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "label": "Stream Error", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_trace_id", - "label": "Session ID", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_ipfrag_num", - "label": "Fragmentation Packets(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_ipfrag_num", - "label": "Fragmentation Packets(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_tcp_lostlen", - "label": "Sequence Gap Loss(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_tcp_lostlen", - "label": "Sequence Gap Loss(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_tcp_unorder_num", - "label": "Unorder Packets(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_tcp_unorder_num", - "label": "Unorder Packets(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_tcp_client_isn", - "label": "TCP Client ISN", - "type": "long", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_tcp_server_isn", - "label": "TCP Server ISN", - "type": "long", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_first_ttl", - "label": "First TTL", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_processing_time", - "label": "Processing Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}", - "type": "long" - }, - { - "name": "http_url", - "label": "Http.URL", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "http_host", - "label": "Http.Host", - "type": "string", - "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "http_domain", - "label": "Http.Domain", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "http_request_line", - "label": "Http.Request Line", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_response_line", - "label": "Http.Response Line", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_request_header", - "label": "Http.Request Header", - "type": "string" - }, - { - "name": "http_response_header", - "label": "Http.Response Header", - "type": "string" - }, - { - "name": "http_request_content", - "label": "Http.Request Content", - "doc": "{\"visibility\":\"disabled\"}", - "type": "string" - }, - { - "name": "http_response_content", - "label": "Http.Response Content", - "doc": "{\"visibility\":\"disabled\"}", - "type": "string" - }, - { - "name": "http_request_body", - "label": "Http.Request Body", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_response_body", - "label": "Http.Response Body", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_request_body_key", - "label": "Http.Request Body Key", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_response_body_key", - "label": "Http.Response Body Key", - "type":"string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_proxy_flag", - "label": "Http.Proxy Flag", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_sequence", - "label": "Http.Sequence", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_snapshot", - "label": "Http.Snapshot", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_cookie", - "label": "Http.Cookie", - "type": "string" - }, - { - "name": "http_referer", - "label": "Http.Referer", - "type": "string" - }, - { - "name": "http_user_agent", - "label": "Http.User Agent", - "type": "string" - }, - { - "name": "http_content_length", - "label": "Http.Content Length", - "type": "string" - }, - { - "name": "http_content_type", - "label": "Http.Content Type", - "type": "string" - }, - { - "name": "http_set_cookie", - "label": "Http.Set Cookie", - "type": "string" - }, - { - "name": "http_version", - "label": "Http.Version", - "type": "string" - }, - { - "name": "http_response_lantency_ms", - "label": "Http.Response Latency(ms)", - "type": "int" - }, - { - "name": "http_session_duration_ms", - "label": "Http.Session Duration(ms)", - "type": "int" - }, - { - "name": "http_action_file_size", - "label": "Http.Action File Size", - "type": "int" - }, - { - "name": "doh_url", - "label": "DoH.URL", - "type": "string" - }, - { - "name": "doh_host", - "label": "DoH.Host", - "type": "string" - }, - { - "name": "doh_request_line", - "label": "DoH.Request Line", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "doh_response_line", - "label": "DoH.Response Line", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "doh_cookie", - "label": "DoH.Cookie", - "type": "string" - }, - { - "name": "doh_referer", - "label": "DoH.Referer", - "type": "string" - }, - { - "name": "doh_user_agent", - "label": "DoH.User Agent", - "type": "string" - }, - { - "name": "doh_content_length", - "label": "DoH.Content Length", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "doh_content_type", - "label": "DoH.Content Type", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "doh_set_cookie", - "label": "DoH.Set Cookie", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "doh_version", - "label": "DoH.Version", - "type": "string" - }, - { - "name": "doh_message_id", - "label": "DoH.Message ID", - "type": "int" - }, - { - "name": "doh_qr", - "label": "DoH.QR", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"REESPONSE\"}]}" - }, - { - "name": "doh_opcode", - "label": "DoH.OPCODE", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"IQUERY\"},{\"code\":\"2\",\"value\":\"STATUS\"},{\"code\":\"5\",\"value\":\"UPDATE\"}]}" - }, - { - "name": "doh_aa", - "label": "DoH.AA", - "type": "int" - }, - { - "name": "doh_tc", - "label": "DoH.TC", - "type": "int" - }, - { - "name": "doh_rd", - "label": "DoH.RD", - "type": "int" - }, - { - "name": "doh_ra", - "label": "DoH.RA", - "type": "int" - }, - { - "name": "doh_rcode", - "label": "DoH.RCODE", - "type": "int" - }, - { - "name": "doh_qdcount", - "label": "DoH.QDCOUNT", - "type": "int" - }, - { - "name": "doh_ancount", - "label": "DoH.ANCOUNT", - "type": "int" - }, - { - "name": "doh_nscount", - "label": "DoH.NSCOUNT", - "type": "int" - }, - { - "name": "doh_arcount", - "label": "DoH.ARCOUNT", - "type": "int" - }, - { - "name": "doh_qname", - "label": "DoH.QNAME", - "type": "string" - }, - { - "name": "doh_qtype", - "label": "DoH.QTYPE", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"A\"},{\"code\":\"2\",\"value\":\"NS\"},{\"code\":\"5\",\"value\":\"CNAME\"},{\"code\":\"6\",\"value\":\"SOA\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"12\",\"value\":\"PTR\"},{\"code\":\"13\",\"value\":\"HINFO\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"15\",\"value\":\"MX\"},{\"code\":\"28\",\"value\":\"AAAA\"}]}" - }, - { - "name": "doh_qclass", - "label": "DoH.QCLASS", - "type": "int" - }, - { - "name": "doh_cname", - "label": "DoH.CNAME", - "type": "string" - }, - { - "name": "doh_sub", - "label": "DoH.SUB", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"DNS\"},{\"code\":\"2\",\"value\":\"DNSSEC\"}]}" - }, - { - "name": "doh_rr", - "label": "DoH.RR", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/proxy_ip_info.avsc b/config/avro/clickhouse/proxy_ip_info.avsc deleted file mode 100644 index 42e944de..00000000 --- a/config/avro/clickhouse/proxy_ip_info.avsc +++ /dev/null @@ -1,22 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "proxy_ip_info", - "fields": [ - { - "name": "stat_time", - "type": "int" - }, - { - "name": "policy_id", - "type": "long" - }, - { - "name": "ip_list", - "type": { - "type": "array", - "items": "string" - } - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/query_log.avsc b/config/avro/clickhouse/query_log.avsc deleted file mode 100644 index 4f5e8d55..00000000 --- a/config/avro/clickhouse/query_log.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace": "system", - "type": "record", - "name": "query_log", - "fields": [ - { - "name": "query_id", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/query_log_cluster.avsc b/config/avro/clickhouse/query_log_cluster.avsc deleted file mode 100644 index d6e75831..00000000 --- a/config/avro/clickhouse/query_log_cluster.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace": "system", - "type": "record", - "name": "query_log_cluster", - "fields": [ - { - "name": "type", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/radius_onff_log.avsc b/config/avro/clickhouse/radius_onff_log.avsc deleted file mode 100644 index 9201ebb7..00000000 --- a/config/avro/clickhouse/radius_onff_log.avsc +++ /dev/null @@ -1,37 +0,0 @@ -{ - "type": "record", - "name": "radius_onff_log", - "namespace": "tsg_galaxy_v3", - "fields": [ - { - "name": "event_timestamp", - "label": "Event Time", - "type": "long" - }, - { - "name": "account", - "label": "Account", - "type": "string" - }, - { - "name": "framed_ip", - "label": "Framed IP", - "type": "string" - }, - { - "name": "acct_session_id", - "label": "Acct Session ID", - "type": "string" - }, - { - "name": "acct_status_type", - "label": "Acct Status Type", - "type": "int" - }, - { - "name": "acct_session_time", - "label": "Acct Session Time", - "type": "int" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/radius_record_log.avsc b/config/avro/clickhouse/radius_record_log.avsc deleted file mode 100644 index 4add512c..00000000 --- a/config/avro/clickhouse/radius_record_log.avsc +++ /dev/null @@ -1,515 +0,0 @@ -{ - "type": "record", - "name": "radius_record_log", - "namespace": "tsg_galaxy_v3", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"functions\":{\"aggregation\":[{\"name\":\"COUNT\",\"function\":\"count(expr)\"},{\"name\":\"COUNT_DISTINCT\",\"function\":\"count(distinct expr)\"},{\"name\":\"AVG\",\"function\":\"avg(expr)\"},{\"name\":\"SUM\",\"function\":\"sum(expr)\"},{\"name\":\"MAX\",\"function\":\"max(expr)\"},{\"name\":\"MIN\",\"function\":\"min(expr)\"}],\"operator\":[{\"name\":\"=\",\"function\":\"expr = value\"},{\"name\":\"!=\",\"function\":\"expr != value\"},{\"name\":\">\",\"function\":\"expr > value\"},{\"name\":\"<\",\"function\":\"expr < value\"},{\"name\":\">=\",\"function\":\"expr >= value\"},{\"name\":\"<=\",\"function\":\"expr <= value\"},{\"name\":\"in\",\"function\":\"expr in (values)\"},{\"name\":\"not in\",\"function\":\"expr not in (values)\"},{\"name\":\"like\",\"function\":\"expr like value\"},{\"name\":\"not like\",\"function\":\"expr not like value\"},{\"name\":\"not empty\",\"function\":\"notEmpty(expr)\"},{\"name\":\"empty\",\"function\":\"empty(expr)\"}]},\"schema_query\":{\"dimensions\":[\"radius_nas_ip\",\"radius_framed_ip\",\"common_subscriber_id\"],\"metrics\":[\"radius_framed_ip\",\"radius_event_timestamp\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\"],\"filters\":[\"radius_framed_ip\",\"common_subscriber_id\",\"radius_packet_type\",\"radius_acct_session_id\",\"radius_acct_multi_session_id\",\"radius_acct_status_type\"],\"references\":{\"aggregation\":[{\"type\":\"int\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"long\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"float\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"double\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"string\",\"functions\":\"COUNT,COUNT_DISTINCT\"},{\"type\":\"date\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"},{\"type\":\"timestamp\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"}],\"operator\":[{\"type\":\"int\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"long\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"float\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"double\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"string\",\"functions\":\"=,!=,in,not in,like,not like,not empty,empty\"},{\"type\":\"date\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"timestamp\",\"functions\":\"=,!=,>,<,>=,<=\"}]},\"details\":{\"general\":[\"common_recv_time\",\"common_log_id\",\"common_stream_trace_id\",\"common_direction\",\"common_stream_dir\",\"common_start_time\",\"common_end_time\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"common_processing_time\"],\"action\":[\"common_action\",\"common_sub_action\",\"common_policy_id\",\"common_user_tags\",\"common_user_region\"],\"source\":[\"common_client_ip\",\"common_internal_ip\",\"common_client_port\",\"common_client_location\",\"common_client_asn\",\"common_entrance_id\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_data_center\",\"common_sled_ip\",\"common_tcp_client_isn\",\"common_subscriber_id\"],\"destination\":[\"common_server_ip\",\"common_external_ip\",\"common_server_port\",\"common_server_location\",\"common_server_asn\",\"common_tcp_server_isn\"],\"application\":[\"common_app_id\",\"common_app_label\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_protocol_label\",\"common_service\",\"common_l4_protocol\"],\"data\":[\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_first_ttl\"]}},\"schema_type\":{\"RADIUS\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"radius_packet_type\",\"radius_nas_ip\",\"radius_framed_ip\",\"radius_account\",\"radius_session_timeout\",\"radius_idle_timeout\",\"radius_acct_status_type\",\"radius_acct_terminate_cause\",\"radius_event_timestamp\",\"radius_nas_port\",\"radius_service_type\",\"radius_framed_protocol\",\"radius_callback_number\",\"radius_callback_id\",\"radius_termination_action\",\"radius_called_station_id\",\"radius_calling_station_id\",\"radius_acct_delay_time\",\"radius_acct_session_id\",\"radius_acct_multi_session_id\",\"radius_acct_input_octets\",\"radius_acct_output_octets\",\"radius_acct_input_packets\",\"radius_acct_output_packets\",\"radius_acct_session_time\",\"radius_acct_link_count\",\"radius_acct_interim_interval\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"radius_nas_ip\",\"radius_framed_ip\",\"radius_acct_status_type\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"radius_nas_ip\",\"radius_framed_ip\",\"radius_acct_status_type\"]}", - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_log_id", - "label": "Log ID", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "type": "long", - "doc":"{\"visibility\":\"hidden\"}" - }, - { - "name": "common_subscriber_id", - "label": "Subscriber ID", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "label": "Client IP", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_client_port", - "label": "Client Port", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_internal_ip", - "label": "Internal IP", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_l4_protocol", - "label": "L4 Protocol", - "type": "string" - }, - { - "name": "common_address_type", - "label": "Address Type", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "label": "Server IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_asn\",\"appendTo\":\"common_server_asn\"}}" - }, - { - "name": "common_server_port", - "label": "Server Port", - "type": "int", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_external_ip", - "label": "External IP", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_action", - "label": "Action", - "type": "int", - "doc": "{\"visibility\":\"hidden\",\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"48\",\"value\":\"Manipulation\"},{\"code\":\"128\",\"value\":\"Allow\"}]}" - }, - { - "name": "common_direction", - "label": "Direction", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"69\",\"value\":\"outbound\"},{\"code\":\"73\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "label": "Sled IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "label": "Client Location", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_client_asn", - "label": "Client ASN", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_server_location", - "label": "Server Location", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_server_asn", - "label": "Server ASN", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_sessions", - "label": "Sessions", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_pkt_num", - "label": "Packets Sent", - "type": "long" - }, - { - "name": "common_s2c_pkt_num", - "label": "Packets Received", - "type": "long" - }, - { - "name": "common_c2s_byte_num", - "label": "Bytes Sent", - "type": "long" - }, - { - "name": "common_s2c_byte_num", - "label": "Bytes Received", - "type": "long" - }, - { - "name": "common_service", - "label": "Service", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "label": "Schema Type", - "type": "string", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"BASE\",\"value\":\"BASE\"},{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"MAIL\",\"value\":\"MAIL\"},{\"code\":\"DNS\",\"value\":\"DNS\"},{\"code\":\"SSL\",\"value\":\"SSL\"},{\"code\":\"FTP\",\"value\":\"FTP\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_user_tags", - "label": "User Tags", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "label": "Sub Action", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_user_region", - "label": "User Region", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "label": "Device ID", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_link_id", - "label": "Link ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "label": "ISP", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_device_tag", - "label": "Device Tag", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_data_center", - "label": "Data Center", - "type": "string", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"Nur-sultan\",\"value\":\"Nur-sultan\"},{\"code\":\"Aktau\",\"value\":\"Aktau\"},{\"code\":\"Aktubinsk\",\"value\":\"Aktubinsk\"},{\"code\":\"Almaty\",\"value\":\"Almaty\"},{\"code\":\"Atyrau\",\"value\":\"Atyrau\"},{\"code\":\"Karaganda\",\"value\":\"Karaganda\"},{\"code\":\"Kokshetau\",\"value\":\"Kokshetau\"},{\"code\":\"Kostanay\",\"value\":\"Kostanay\"},{\"code\":\"Kyzylorda\",\"value\":\"Kyzylorda\"},{\"code\":\"Pavlodar\",\"value\":\"Pavlodar\"},{\"code\":\"Petropavl\",\"value\":\"Petropavl\"},{\"code\":\"Semey\",\"value\":\"Semey\"},{\"code\":\"Shymkent\",\"value\":\"Shymkent\"},{\"code\":\"Taldykurgan\",\"value\":\"Taldykurgan\"},{\"code\":\"Taraz\",\"value\":\"Taraz\"},{\"code\":\"Uralsk\",\"value\":\"Uralsk\"},{\"code\":\"Ust-Kamenogorsk\",\"value\":\"Ust-Kamenogorsk\"},{\"code\":\"Zhezkazgan\",\"value\":\"Zhezkazgan\"}],\"allow_query\":\"true\"}" - }, - { - "name": "common_encapsulation", - "label": "Encapsulation", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_label", - "label": "Application Label", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_protocol_label", - "label": "Protocol Label", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_id", - "label": "Application ID", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_surrogate_id", - "label": "Surrogate ID", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_l7_protocol", - "label": "L7 Protocol", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_start_time", - "label": "Start Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"visibility\":\"hidden\"}" - }, - { - "name": "common_end_time", - "label": "End Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"get_value\",\"appendTo\":\"common_recv_time\"},\"visibility\":\"hidden\"}" - }, - { - "name": "common_establish_latency_ms", - "label": "Establish Latency(ms)", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_con_duration_ms", - "label": "Duration(ms)", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_dir", - "label": "Stream Direction", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "label": "Address List", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_has_dup_traffic", - "label": "Duplication Traffic", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "label": "Stream Error", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_trace_id", - "label": "Session ID", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_ipfrag_num", - "label": "Fragmentation Packets(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_ipfrag_num", - "label": "Fragmentation Packets(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_tcp_lostlen", - "label": "Sequence Gap Loss(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_tcp_lostlen", - "label": "Sequence Gap Loss(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_tcp_unorder_num", - "label": "Unorder Packets(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_tcp_unorder_num", - "label": "Unorder Packets(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_tcp_client_isn", - "label": "TCP Client ISN", - "type": "long", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_tcp_server_isn", - "label": "TCP Server ISN", - "type": "long", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_first_ttl", - "label": "First TTL", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_processing_time", - "label": "Processing Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}", - "type": "long" - }, - { - "name": "radius_packet_type", - "label": "Packet Type", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"Access-Request\"},{\"code\":\"2\",\"value\":\"Access-Accept\"},{\"code\":\"3\",\"value\":\"Access-Reject\"},{\"code\":\"4\",\"value\":\"Accounting-Request\"},{\"code\":\"5\",\"value\":\"Accounting-Response\"},{\"code\":\"11\",\"value\":\"Access-Challenge\"}]}" - }, - { - "name": "radius_account", - "label": "Account", - "type": "string", - "doc": "{\"format\":{\"functions\":\"get_value\",\"appendTo\":\"common_subscriber_id\"}}" - }, - { - "name": "radius_nas_ip", - "label": "Nas IP", - "type": "string" - }, - { - "name": "radius_framed_ip", - "label": "Framed IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "radius_session_timeout", - "label": "Session Timeout", - "type": "int" - }, - { - "name": "radius_idle_timeout", - "label": "Idle Timeout", - "type": "int" - }, - { - "name": "radius_acct_status_type", - "label": "ACC Status Type", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"Start\"},{\"code\":\"2\",\"value\":\"Stop\"},{\"code\":\"3\",\"value\":\"Interim-Update\"},{\"code\":\"7\",\"value\":\"Accounting-On\"},{\"code\":\"8\",\"value\":\"Accounting-Off\"}]}" - }, - { - "name": "radius_acct_terminate_cause", - "label": "Acct Terminate Cause", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"User Request\"},{\"code\":\"2\",\"value\":\"Lost Carrier\"},{\"code\":\"3\",\"value\":\"Lost Service\"},{\"code\":\"4\",\"value\":\"Idle Timeout\"},{\"code\":\"5\",\"value\":\"Session Timeout\"},{\"code\":\"6\",\"value\":\"Admin Reset\"},{\"code\":\"7\",\"value\":\"Admin Reboot\"},{\"code\":\"8\",\"value\":\"Port Error\"},{\"code\":\"9\",\"value\":\"NAS Error\"},{\"code\":\"10\",\"value\":\"NAS Request\"},{\"code\":\"11\",\"value\":\"NAS Reboot\"},{\"code\":\"12\",\"value\":\"Port Unneeded\"},{\"code\":\"13\",\"value\":\"Port Preempted\"},{\"code\":\"14\",\"value\":\"Port Suspended\"},{\"code\":\"15\",\"value\":\"Service Unavailable\"},{\"code\":\"16\",\"value\":\"Callback\"},{\"code\":\"17\",\"value\":\"User Error\"},{\"code\":\"18\",\"value\":\"Host Request\"}]}" - }, - { - "name": "radius_event_timestamp", - "label": "Event Timestamp", - "type": "int" - }, - { - "name": "radius_service_type", - "label": "Service Type", - "type": "int" - }, - { - "name": "radius_nas_port", - "label": "Nas Port", - "type": "int" - }, - { - "name": "radius_framed_protocol", - "label": "Framed Protocol", - "type": "int" - }, - { - "name": "radius_callback_number", - "label": "Callback Number", - "type": "string" - }, - { - "name": "radius_callback_id", - "label": "Callback ID", - "type": "string" - }, - { - "name": "radius_termination_action", - "label": "Termination Action", - "type": "int" - }, - { - "name": "radius_called_station_id", - "label": "Called Station Id", - "type": "string" - }, - { - "name": "radius_calling_station_id", - "label": "Calling Station Id", - "type": "string" - }, - { - "name": "radius_acct_delay_time", - "label": "Acct Delay Time", - "type": "int" - }, - { - "name": "radius_acct_session_id", - "label": "Acct Session ID", - "type": "string" - }, - { - "name": "radius_acct_multi_session_id", - "label": "Acct Multi Session ID", - "type": "string" - }, - { - "name": "radius_acct_input_octets", - "label": "Acct Input Octets", - "type": "long" - }, - { - "name": "radius_acct_output_octets", - "label": "Acct Output Octets", - "type": "long" - }, - { - "name": "radius_acct_input_packets", - "label": "Acct Input Packets", - "type": "long" - }, - { - "name": "radius_acct_output_packets", - "label": "Acct Output Packets", - "type": "long" - }, - { - "name": "radius_acct_session_time", - "label": "Acct Session Time", - "type": "int" - }, - { - "name": "radius_acct_link_count", - "label": "Acct Link Count", - "type": "int" - }, - { - "name": "radius_acct_interim_interval", - "label": "Acct Interim Interval", - "type": "int" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/security_event_log.avsc b/config/avro/clickhouse/security_event_log.avsc deleted file mode 100644 index 483592f4..00000000 --- a/config/avro/clickhouse/security_event_log.avsc +++ /dev/null @@ -1,987 +0,0 @@ -{ - "type": "record", - "name": "security_event_log", - "namespace": "tsg_galaxy_v3", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"functions\":{\"aggregation\":[{\"name\":\"COUNT\",\"function\":\"count(expr)\"},{\"name\":\"COUNT_DISTINCT\",\"function\":\"count(distinct expr)\"},{\"name\":\"AVG\",\"function\":\"avg(expr)\"},{\"name\":\"SUM\",\"function\":\"sum(expr)\"},{\"name\":\"MAX\",\"function\":\"max(expr)\"},{\"name\":\"MIN\",\"function\":\"min(expr)\"}],\"operator\":[{\"name\":\"=\",\"function\":\"expr = value\"},{\"name\":\"!=\",\"function\":\"expr != value\"},{\"name\":\">\",\"function\":\"expr > value\"},{\"name\":\"<\",\"function\":\"expr < value\"},{\"name\":\">=\",\"function\":\"expr >= value\"},{\"name\":\"<=\",\"function\":\"expr <= value\"},{\"name\":\"in\",\"function\":\"expr in (values)\"},{\"name\":\"not in\",\"function\":\"expr not in (values)\"},{\"name\":\"like\",\"function\":\"expr like value\"},{\"name\":\"not like\",\"function\":\"expr not like value\"},{\"name\":\"not empty\",\"function\":\"notEmpty(expr)\"},{\"name\":\"empty\",\"function\":\"empty(expr)\"}]},\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_policy_id\",\"common_action\",\"common_sled_ip\",\"common_device_id\",\"common_client_location\",\"common_server_location\",\"common_subscriber_id\",\"common_client_port\",\"common_server_port\",\"common_schema_type\",\"common_l4_protocol\",\"common_l7_protocol\",\"common_data_center\",\"common_client_asn\",\"common_server_asn\",\"common_start_time\",\"common_end_time\",\"http_host\",\"http_domain\",\"http_url\",\"ssl_sni\",\"ssl_ja3_hash\",\"ssl_client_side_version\",\"ssl_server_side_version\",\"mail_account\",\"mail_from\",\"mail_to\",\"quic_sni\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_subscriber_id\",\"common_sled_ip\",\"common_device_id\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"http_host\",\"http_domain\",\"http_url\",\"ssl_sni\",\"ssl_ja3_hash\",\"ssl_client_side_latency\",\"ssl_server_side_latency\",\"mail_account\",\"mail_from\",\"mail_to\",\"quic_sni\"],\"filters\":[\"common_policy_id\",\"common_action\",\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_client_port\",\"common_server_port\",\"common_client_location\",\"common_server_location\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_l4_protocol\",\"common_l7_protocol\",\"common_stream_dir\",\"common_data_center\",\"common_sled_ip\",\"common_device_id\",\"common_direction\",\"common_schema_type\",\"common_client_asn\",\"common_server_asn\",\"common_start_time\",\"common_end_time\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"http_host\",\"http_domain\",\"http_url\",\"http_content_type\",\"ssl_sni\",\"ssl_ja3_hash\",\"ssl_pinningst\",\"ssl_intercept_state\",\"ssl_client_side_version\",\"ssl_server_side_version\",\"ssl_cert_verify\",\"ssl_client_side_latency\",\"ssl_server_side_latency\",\"mail_account\",\"mail_from\",\"mail_to\",\"mail_subject\",\"quic_sni\"],\"references\":{\"aggregation\":[{\"type\":\"int\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"long\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"float\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"double\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"string\",\"functions\":\"COUNT,COUNT_DISTINCT\"},{\"type\":\"date\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"},{\"type\":\"timestamp\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"}],\"operator\":[{\"type\":\"int\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"long\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"float\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"double\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"string\",\"functions\":\"=,!=,in,not in,like,not like,not empty,empty\"},{\"type\":\"date\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"timestamp\",\"functions\":\"=,!=,>,<,>=,<=\"}]},\"details\":{\"general\":[\"common_recv_time\",\"common_log_id\",\"common_stream_trace_id\",\"common_direction\",\"common_stream_dir\",\"common_start_time\",\"common_end_time\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"common_processing_time\"],\"action\":[\"common_action\",\"common_sub_action\",\"common_policy_id\",\"common_user_tags\",\"common_user_region\"],\"source\":[\"common_client_ip\",\"common_internal_ip\",\"common_client_port\",\"common_client_location\",\"common_client_asn\",\"common_entrance_id\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_data_center\",\"common_sled_ip\",\"common_tcp_client_isn\",\"common_subscriber_id\"],\"destination\":[\"common_server_ip\",\"common_external_ip\",\"common_server_port\",\"common_server_location\",\"common_server_asn\",\"common_tcp_server_isn\"],\"application\":[\"common_app_id\",\"common_app_label\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_protocol_label\",\"common_service\",\"common_l4_protocol\"],\"data\":[\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_first_ttl\"]}},\"schema_type\":{\"BASE\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\"]},\"HTTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"http_url\",\"http_host\",\"http_domain\",\"http_request_line\",\"http_response_line\",\"http_request_header\",\"http_response_header\",\"http_request_content\",\"http_response_content\",\"http_request_body\",\"http_response_body\",\"http_request_body_key\",\"http_response_body_key\",\"http_proxy_flag\",\"http_sequence\",\"http_snapshot\",\"http_cookie\",\"http_referer\",\"http_user_agent\",\"http_content_length\",\"http_content_type\",\"http_set_cookie\",\"http_version\",\"http_response_lantency_ms\",\"http_session_duration_ms\",\"http_action_file_size\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"http_url\",\"common_server_port\"]},\"MAIL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"mail_protocol_type\",\"mail_account\",\"mail_from_cmd\",\"mail_to_cmd\",\"mail_from\",\"mail_to\",\"mail_cc\",\"mail_bcc\",\"mail_subject\",\"mail_subject_charset\",\"mail_content\",\"mail_content_charset\",\"mail_attachment_name\",\"mail_attachment_name_charset\",\"mail_attachment_content\",\"mail_eml_file\",\"mail_snapshot\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"mail_from\",\"mail_to\",\"mail_subject\"]},\"DNS\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"dns_message_id\",\"dns_qr\",\"dns_opcode\",\"dns_aa\",\"dns_tc\",\"dns_rd\",\"dns_ra\",\"dns_rcode\",\"dns_qdcount\",\"dns_ancount\",\"dns_nscount\",\"dns_arcount\",\"dns_qname\",\"dns_qtype\",\"dns_qclass\",\"dns_cname\",\"dns_sub\",\"dns_rr\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_client_ip\",\"dns_qr\",\"dns_qname\",\"dns_qtype\"]},\"SSL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"ssl_sni\",\"ssl_san\",\"ssl_cn\",\"ssl_pinningst\",\"ssl_intercept_state\",\"ssl_server_side_latency\",\"ssl_client_side_latency\",\"ssl_server_side_version\",\"ssl_client_side_version\",\"ssl_cert_verify\",\"ssl_error\",\"ssl_con_latency_ms\",\"ssl_ja3_fingerprint\",\"ssl_ja3_hash\",\"ssl_cert_issuer\",\"ssl_cert_subject\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"ssl_sni\",\"common_server_ip\",\"common_server_port\"]},\"QUIC\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"quic_version\",\"quic_sni\",\"quic_user_agent\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"quic_sni\",\"common_server_ip\",\"common_server_port\"]},\"FTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"ftp_account\",\"ftp_url\",\"ftp_content\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"ftp_url\",\"common_server_ip\",\"common_server_port\"]},\"BGP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"bgp_type\",\"bgp_as_num\",\"bgp_route\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"bgp_type\",\"bgp_as_num\",\"common_server_ip\",\"common_server_port\"]},\"SIP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"sip_call_id\",\"sip_from\",\"sip_to\",\"sip_user_agent\",\"sip_server\",\"sip_from_sdp_connect_ip\",\"sip_from_sdp_media_port\",\"sip_from_sdp_media_type\",\"sip_from_sdp_content\",\"sip_to_sdp_connect_ip\",\"sip_to_sdp_media_port\",\"sip_to_sdp_media_type\",\"sip_to_sdp_content\",\"sip_duration\",\"sip_bye\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"sip_from\",\"sip_to\",\"sip_call_id\",\"common_server_ip\",\"common_server_port\"]},\"RTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"rtp_payload_type_c2s\",\"rtp_payload_type_s2c\",\"rtp_pcap_dir_c2s\",\"rtp_pcap_dir_s2c\",\"rtp_from_direction\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"rtp_pcap_dir_c2s\",\"rtp_pcap_dir_s2c\",\"rtp_from_direction\"]},\"APP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"app_extra_info\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_app_id\",\"common_app_label\",\"app_extra_info\",\"common_server_ip\",\"common_server_port\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"common_schema_type\"]}", - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_log_id", - "label": "Log ID", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_subscriber_id", - "label": "Subscriber ID", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "label": "Client IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_asn,radius_match\",\"appendTo\":\"common_client_asn,common_subscriber_id\"}}" - }, - { - "name": "common_internal_ip", - "label": "Internal IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"if\",\"param\":\"$.common_direction=69,$.common_client_ip,$.common_server_ip\"}}" - }, - { - "name": "common_client_port", - "label": "Client Port", - "type": "int", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_l4_protocol", - "label": "L4 Protocol", - "type": "string" - }, - { - "name": "common_address_type", - "label": "Address Type", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "label": "Server IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_asn\",\"appendTo\":\"common_server_asn\"}}" - }, - { - "name": "common_server_port", - "label": "Server Port", - "type": "int", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_external_ip", - "label": "External IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"if\",\"param\":\"$.common_direction=73,$.common_client_ip,$.common_server_ip\"}}" - }, - { - "name": "common_action", - "label": "Action", - "type": "int", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"128\",\"value\":\"Allow\"}]}" - }, - { - "name": "common_direction", - "label": "Direction", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"69\",\"value\":\"outbound\"},{\"code\":\"73\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "label": "Sled IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "label": "Client Location", - "type": "string" - }, - { - "name": "common_client_asn", - "label": "Client ASN", - "type": "string" - }, - { - "name": "common_server_location", - "label": "Server Location", - "type": "string" - }, - { - "name": "common_server_asn", - "label": "Server ASN", - "type": "string" - }, - { - "name": "common_sessions", - "label": "Sessions", - "type": "long", - "doc": "{\"format\":{\"functions\":\"set_value\",\"param\":\"1\"}}" - }, - { - "name": "common_c2s_pkt_num", - "label": "Packets Sent", - "type": "long" - }, - { - "name": "common_s2c_pkt_num", - "label": "Packets Received", - "type": "long" - }, - { - "name": "common_c2s_byte_num", - "label": "Bytes Sent", - "type": "long" - }, - { - "name": "common_s2c_byte_num", - "label": "Bytes Received", - "type": "long" - }, - { - "name": "common_service", - "label": "Service", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "label": "Schema Type", - "type": "string", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"MAIL\",\"value\":\"MAIL\"},{\"code\":\"DNS\",\"value\":\"DNS\"},{\"code\":\"SSL\",\"value\":\"SSL\"},{\"code\":\"QUIC\",\"value\":\"QUIC\"},{\"code\":\"FTP\",\"value\":\"FTP\"},{\"code\":\"SIP\",\"value\":\"SIP\"},{\"code\":\"RTP\",\"value\":\"RTP\"},{\"code\":\"APP\",\"value\":\"APP\"}],\"allow_query\":\"true\"}" - }, - { - "name": "common_user_tags", - "label": "User Tags", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "label": "Sub Action", - "type": "string" - }, - { - "name": "common_user_region", - "label": "User Region", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "label": "Device ID", - "type": "string" - }, - { - "name": "common_link_id", - "label": "Link ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "label": "ISP", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_device_tag", - "label": "Device Tag", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_data_center", - "label": "Data Center", - "type": "string", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"Nur-sultan\",\"value\":\"Nur-sultan\"},{\"code\":\"Aktau\",\"value\":\"Aktau\"},{\"code\":\"Aktubinsk\",\"value\":\"Aktubinsk\"},{\"code\":\"Almaty\",\"value\":\"Almaty\"},{\"code\":\"Atyrau\",\"value\":\"Atyrau\"},{\"code\":\"Karaganda\",\"value\":\"Karaganda\"},{\"code\":\"Kokshetau\",\"value\":\"Kokshetau\"},{\"code\":\"Kostanay\",\"value\":\"Kostanay\"},{\"code\":\"Kyzylorda\",\"value\":\"Kyzylorda\"},{\"code\":\"Pavlodar\",\"value\":\"Pavlodar\"},{\"code\":\"Petropavl\",\"value\":\"Petropavl\"},{\"code\":\"Semey\",\"value\":\"Semey\"},{\"code\":\"Shymkent\",\"value\":\"Shymkent\"},{\"code\":\"Taldykurgan\",\"value\":\"Taldykurgan\"},{\"code\":\"Taraz\",\"value\":\"Taraz\"},{\"code\":\"Uralsk\",\"value\":\"Uralsk\"},{\"code\":\"Ust-Kamenogorsk\",\"value\":\"Ust-Kamenogorsk\"},{\"code\":\"Zhezkazgan\",\"value\":\"Zhezkazgan\"}],\"allow_query\":\"true\"}" - }, - { - "name": "common_encapsulation", - "label": "Encapsulation", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_label", - "label": "Application Label", - "type": "string" - }, - { - "name": "common_protocol_label", - "label": "Protocol Label", - "type": "string" - }, - { - "name": "common_app_id", - "label": "Application ID", - "type": "int" - }, - { - "name": "common_app_surrogate_id", - "label": "Surrogate ID", - "type": "int" - }, - { - "name": "common_l7_protocol", - "label": "L7 Protocol", - "type": "string" - }, - { - "name": "common_start_time", - "label": "Start Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_end_time", - "label": "End Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"get_value\",\"appendTo\":\"common_recv_time\"}}" - }, - { - "name": "common_establish_latency_ms", - "label": "Establish Latency(ms)", - "type": "int" - }, - { - "name": "common_con_duration_ms", - "label": "Duration(ms)", - "type": "int" - }, - { - "name": "common_stream_dir", - "label": "Stream Direction", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "label": "Address List", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_has_dup_traffic", - "label": "Duplication Traffic", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "label": "Stream Error", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "label": "Session ID", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_ipfrag_num", - "label": "Fragmentation Packets(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_ipfrag_num", - "label": "Fragmentation Packets(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_tcp_lostlen", - "label": "Sequence Gap Loss(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_tcp_lostlen", - "label": "Sequence Gap Loss(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_tcp_unorder_num", - "label": "Unorder Packets(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_tcp_unorder_num", - "label": "Unorder Packets(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_tcp_client_isn", - "label": "TCP Client ISN", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_tcp_server_isn", - "label": "TCP Server ISN", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_first_ttl", - "label": "First TTL", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_processing_time", - "label": "Processing Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}", - "type": "long" - }, - { - "name": "http_url", - "label": "Http.URL", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "http_host", - "label": "Http.Host", - "type": "string", - "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "http_domain", - "label": "Http.Domain", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "http_request_line", - "label": "Http.Request Line", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_response_line", - "label": "Http.Response Line", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_request_header", - "label": "Http.Request Header", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_header", - "label": "Http.Response Header", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_content", - "label": "Http.Request Content", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_content", - "label": "Http.Response Content", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_body", - "label": "Http.Request Body", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_response_body", - "label": "Http.Response Body", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_request_body_key", - "label": "Http.Request Body Key", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_response_body_key", - "label": "Http.Response Body Key", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_proxy_flag", - "label": "http.Proxy Flag", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_sequence", - "label": "Http.Sequence", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_snapshot", - "label": "Http.Snapshot", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_cookie", - "label": "Http.Cookie", - "type": "string" - }, - { - "name": "http_referer", - "label": "Http.Referer", - "type": "string" - }, - { - "name": "http_user_agent", - "label": "Http.User Agent", - "type": "string" - }, - { - "name": "http_content_length", - "label": "Http.Content Length", - "type": "string" - }, - { - "name": "http_content_type", - "label": "Http.Content Type", - "type": "string" - }, - { - "name": "http_set_cookie", - "label": "Http.Set Cookie", - "type": "string" - }, - { - "name": "http_version", - "label": "Http.Version", - "type": "string" - }, - { - "name": "http_response_lantency_ms", - "label": "Http.Response Latency(ms)", - "type": "int" - }, - { - "name": "http_action_file_size", - "label": "Http.Action File Size", - "type": "int" - }, - { - "name": "http_session_duration_ms", - "label": "Http.Session Duration(ms)", - "type": "int" - }, - { - "name": "mail_protocol_type", - "label": "Mail.Protocol Type", - "type": "string" - }, - { - "name": "mail_account", - "label": "Mail.Account", - "type": "string" - }, - { - "name": "mail_from_cmd", - "label": "Mail.From CMD", - "type": "string" - }, - { - "name": "mail_to_cmd", - "label": "Mail.To CMD", - "type": "string" - }, - { - "name": "mail_from", - "label": "Mail.From", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"email\"}}" - }, - { - "name": "mail_to", - "label": "Mail.To", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"email\"}}" - }, - { - "name": "mail_cc", - "label": "Mail.CC", - "type": "string" - }, - { - "name": "mail_bcc", - "label": "Mail.BCC", - "type": "string" - }, - { - "name": "mail_subject", - "label": "Mail.Subject", - "type": "string", - "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"$.mail_subject_charset\"}}" - }, - { - "name": "mail_subject_charset", - "label": "Mail.Subject Charset", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_content", - "label": "Mail.Content", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_content_charset", - "label": "Mail.Content Charset", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_attachment_name", - "label": "Mail.Attachment", - "type": "string", - "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"$.mail_attachment_name_charset\"}}" - }, - { - "name": "mail_attachment_name_charset", - "label": "Mail.Attachment Charset", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_attachment_content", - "label": "Mail.Attachment Content", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_eml_file", - "label": "Mail.EML File", - "type": "string", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "mail_snapshot", - "label": "Mail.Snapshot", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "dns_message_id", - "label": "Dns.Message ID", - "type": "int" - }, - { - "name": "dns_qr", - "label": "Dns.QR", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"REESPONSE\"}]}" - }, - { - "name": "dns_opcode", - "label": "Dns.OPCODE", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"IQUERY\"},{\"code\":\"2\",\"value\":\"STATUS\"},{\"code\":\"5\",\"value\":\"UPDATE\"}]}" - }, - { - "name": "dns_aa", - "label": "Dns.AA", - "type": "int" - }, - { - "name": "dns_tc", - "label": "Dns.TC", - "type": "int" - }, - { - "name": "dns_rd", - "label": "Dns.RD", - "type": "int" - }, - { - "name": "dns_ra", - "label": "Dns.RA", - "type": "int" - }, - { - "name": "dns_rcode", - "label": "Dns.RCODE", - "type": "int" - }, - { - "name": "dns_qdcount", - "label": "Dns.QDCOUNT", - "type": "int" - }, - { - "name": "dns_ancount", - "label": "Dns.ANCOUNT", - "type": "int" - }, - { - "name": "dns_nscount", - "label": "Dns.NSCOUNT", - "type": "int" - }, - { - "name": "dns_arcount", - "label": "Dns.ARCOUNT", - "type": "int" - }, - { - "name": "dns_qname", - "label": "Dns.QNAME", - "type": "string" - }, - { - "name": "dns_qtype", - "label": "Dns.QTYPE", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"A\"},{\"code\":\"2\",\"value\":\"NS\"},{\"code\":\"5\",\"value\":\"CNAME\"},{\"code\":\"6\",\"value\":\"SOA\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"12\",\"value\":\"PTR\"},{\"code\":\"13\",\"value\":\"HINFO\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"15\",\"value\":\"MX\"},{\"code\":\"28\",\"value\":\"AAAA\"}]}" - }, - { - "name": "dns_qclass", - "label": "Dns.QCLASS", - "type": "int" - }, - { - "name": "dns_cname", - "label": "Dns.CNAME", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "dns_sub", - "label": "Dns.SUB", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"DNS\"},{\"code\":\"2\",\"value\":\"DNSSEC\"}]}" - }, - { - "name": "dns_rr", - "label": "Dns.RR", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "ssl_version", - "label": "SSL.Version", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "ssl_sni", - "label": "SSL.SNI", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "ssl_san", - "label": "SSL.SAN", - "type": "string" - }, - { - "name": "ssl_cn", - "label": "SSL.CN", - "type": "string" - }, - { - "name": "ssl_pinningst", - "label": "SSL.Pinning", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"Not Pinning\"},{\"code\":\"1\",\"value\":\"Pinning\"},{\"code\":\"2\",\"value\":\"Maybe Pinning\"}]}" - }, - { - "name": "ssl_intercept_state", - "label": "SSL.Intercept State", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"Passthrough\"},{\"code\":\"1\",\"value\":\"Intercept\"},{\"code\":\"2\",\"value\":\"Shutdown\"}]}" - }, - { - "name": "ssl_server_side_latency", - "label": "SSL.Server Side Latency(ms)", - "type": "int" - }, - { - "name": "ssl_client_side_latency", - "label": "SSL.Client Side Latency(ms)", - "type": "int" - }, - { - "name": "ssl_server_side_version", - "label": "SSL.Server Side Version", - "type": "string" - }, - { - "name": "ssl_client_side_version", - "label": "SSL.Client Side Version", - "type": "string" - }, - { - "name": "ssl_cert_verify", - "label": "SSL.Certificate Verify", - "type": "int", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}]}" - }, - { - "name": "ssl_error", - "label": "SSL.Error", - "type": "string" - }, - { - "name": "ssl_con_latency_ms", - "label": "SSL.Connection Latency(ms)", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "ssl_ja3_fingerprint", - "label": "SSL.JA3", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "ssl_ja3_hash", - "label": "SSL.JA3 hash", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "ssl_cert_issuer", - "label": "SSL.Issuer", - "type": "string" - }, - { - "name": "ssl_cert_subject", - "label": "SSL.Subject", - "type": "string" - }, - { - "name": "quic_version", - "label": "Quic.Version", - "type": "string" - }, - { - "name": "quic_sni", - "label": "Quic.SNI", - "type": "string", - "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "quic_user_agent", - "label": "Quic.User Agent", - "type": "string" - }, - { - "name": "ftp_account", - "label": "Ftp.Account", - "type": "string" - }, - { - "name": "ftp_url", - "label": "Ftp.URL", - "type": "string" - }, - { - "name": "ftp_content", - "label": "Ftp.Content", - "type": "string" - }, - { - "name": "bgp_type", - "label": "BGP.Type", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "bgp_as_num", - "label": "BGP.AS Number", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "bgp_route", - "label": "BGP.Route", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_calling_account", - "label": "Voip.Calling Account", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_called_account", - "label": "Voip.Called Account", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_calling_number", - "label": "Voip.Calling Number", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_called_number", - "label": "Voip.Called Number", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "streaming_media_url", - "label": "Streaming.Media URL", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "streaming_media_protocol", - "label": "Streaming.Media Protocol", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "app_extra_info", - "label": "APP.Extra Info", - "type": "string" - }, - { - "name": "sip_call_id", - "label": "SIP.Call-ID", - "type": "string" - }, - { - "name": "sip_from", - "label": "SIP.From", - "type": "string" - }, - { - "name": "sip_to", - "label": "SIP.To", - "type": "string" - }, - { - "name": "sip_user_agent", - "label": "SIP.User-Agent", - "type": "string" - }, - { - "name": "sip_server", - "label": "SIP.Server", - "type": "string" - }, - { - "name": "sip_from_sdp_connect_ip", - "label": "SIP.From Connect IP", - "type": "string" - }, - { - "name": "sip_from_sdp_media_port", - "label": "SIP.From Media Port", - "type": "int" - }, - { - "name": "sip_from_sdp_media_type", - "label": "SIP.From Media Type", - "type": "string" - }, - { - "name": "sip_from_sdp_content", - "label": "SIP.From SDP Content", - "type": "string" - }, - { - "name": "sip_to_sdp_connect_ip", - "label": "SIP.To Connect IP", - "type": "string" - }, - { - "name": "sip_to_sdp_media_port", - "label": "SIP.To Media Port", - "type": "int" - }, - { - "name": "sip_to_sdp_media_type", - "label": "SIP.To Media Type", - "type": "string" - }, - { - "name": "sip_to_sdp_content", - "label": "SIP.To SDP Content", - "type": "string" - }, - { - "name": "sip_duration", - "label": "SIP.Duration", - "type": "int" - }, - { - "name": "sip_bye", - "label": "SIP.Bye", - "type": "string" - }, - { - "name": "rtp_payload_type_c2s", - "label": "RTP.Payload(c2s)", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"PCMU\"},{\"code\":\"1\",\"value\":\"1016\"},{\"code\":\"2\",\"value\":\"G721\"},{\"code\":\"3\",\"value\":\"GSM\"},{\"code\":\"4\",\"value\":\"G723\"},{\"code\":\"5\",\"value\":\"DVI4_8000\"},{\"code\":\"6\",\"value\":\"DVI4_16000\"},{\"code\":\"7\",\"value\":\"LPC\"},{\"code\":\"8\",\"value\":\"PCMA\"},{\"code\":\"9\",\"value\":\"G722\"},{\"code\":\"10\",\"value\":\"L16_STEREO\"},{\"code\":\"11\",\"value\":\"L16_MONO\"},{\"code\":\"12\",\"value\":\"QCELP\"},{\"code\":\"13\",\"value\":\"CN\"},{\"code\":\"14\",\"value\":\"MPA\"},{\"code\":\"15\",\"value\":\"G728\"},{\"code\":\"16\",\"value\":\"DVI4_11025\"},{\"code\":\"17\",\"value\":\"DVI4_22050\"},{\"code\":\"18\",\"value\":\"G729\"},{\"code\":\"19\",\"value\":\"CN_OLD\"},{\"code\":\"25\",\"value\":\"CELB\"},{\"code\":\"26\",\"value\":\"JPEG\"},{\"code\":\"28\",\"value\":\"NV\"},{\"code\":\"31\",\"value\":\"H261\"},{\"code\":\"32\",\"value\":\"MPV\"},{\"code\":\"33\",\"value\":\"MP2T\"},{\"code\":\"34\",\"value\":\"H263\"}]}", - "type": "int" - }, - { - "name": "rtp_payload_type_s2c", - "label": "RTP.Payload(s2c)", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"PCMU\"},{\"code\":\"1\",\"value\":\"1016\"},{\"code\":\"2\",\"value\":\"G721\"},{\"code\":\"3\",\"value\":\"GSM\"},{\"code\":\"4\",\"value\":\"G723\"},{\"code\":\"5\",\"value\":\"DVI4_8000\"},{\"code\":\"6\",\"value\":\"DVI4_16000\"},{\"code\":\"7\",\"value\":\"LPC\"},{\"code\":\"8\",\"value\":\"PCMA\"},{\"code\":\"9\",\"value\":\"G722\"},{\"code\":\"10\",\"value\":\"L16_STEREO\"},{\"code\":\"11\",\"value\":\"L16_MONO\"},{\"code\":\"12\",\"value\":\"QCELP\"},{\"code\":\"13\",\"value\":\"CN\"},{\"code\":\"14\",\"value\":\"MPA\"},{\"code\":\"15\",\"value\":\"G728\"},{\"code\":\"16\",\"value\":\"DVI4_11025\"},{\"code\":\"17\",\"value\":\"DVI4_22050\"},{\"code\":\"18\",\"value\":\"G729\"},{\"code\":\"19\",\"value\":\"CN_OLD\"},{\"code\":\"25\",\"value\":\"CELB\"},{\"code\":\"26\",\"value\":\"JPEG\"},{\"code\":\"28\",\"value\":\"NV\"},{\"code\":\"31\",\"value\":\"H261\"},{\"code\":\"32\",\"value\":\"MPV\"},{\"code\":\"33\",\"value\":\"MP2T\"},{\"code\":\"34\",\"value\":\"H263\"}]}", - "type": "int" - }, - { - "name": "rtp_pcap_dir_c2s", - "label": "RTP.PCAP(c2s)", - "doc": "{\"constraints\":{\"type\":\"file\"}}", - "type": "string" - }, - { - "name": "rtp_pcap_dir_s2c", - "label": "RTP.PCAP(s2c)", - "doc": "{\"constraints\":{\"type\":\"file\"}}", - "type": "string" - }, - { - "name": "rtp_from_direction", - "label": "Direction", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"unknown\"},{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"}]}", - "type": "int" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/security_ip_info.avsc b/config/avro/clickhouse/security_ip_info.avsc deleted file mode 100644 index 17443263..00000000 --- a/config/avro/clickhouse/security_ip_info.avsc +++ /dev/null @@ -1,33 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "security_ip_info", - "fields": [ - { - "name": "stat_time", - "type": "int" - }, - { - "name": "policy_id", - "type": "long" - }, - { - "name": "ip", - "type": "string" - }, - { - "name": "domain_list", - "type": { - "type": "array", - "items": "string" - } - }, - { - "name": "port_list", - "type": { - "type": "array", - "items": "int" - } - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/security_website_domain_info.avsc b/config/avro/clickhouse/security_website_domain_info.avsc deleted file mode 100644 index 02a942ee..00000000 --- a/config/avro/clickhouse/security_website_domain_info.avsc +++ /dev/null @@ -1,47 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "security_website_domain_info", - "fields": [ - { - "name": "stat_time", - "type": "int" - }, - { - "name": "policy_id", - "type": "long" - }, - { - "name": "domain", - "type": "string" - }, - { - "name": "ip_list", - "type": { - "type": "array", - "items": "string" - } - }, - { - "name": "cdn_list", - "type": { - "type": "array", - "items": "string" - } - }, - { - "name": "protocol_type_list", - "type": { - "type": "array", - "items": "string" - } - }, - { - "name": "port_list", - "type": { - "type": "array", - "items": "int" - } - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/sys_packet_capture_log.avsc b/config/avro/clickhouse/sys_packet_capture_log.avsc deleted file mode 100644 index c0a9cbba..00000000 --- a/config/avro/clickhouse/sys_packet_capture_log.avsc +++ /dev/null @@ -1,389 +0,0 @@ -{ - "type": "record", - "name": "sys_packet_capture_log", - "namespace": "tsg_galaxy_v3", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\"}", - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}" - }, - { - "name": "common_log_id", - "label": "Log ID", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_subscriber_id", - "label": "Subscriber ID", - "type": "string", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "label": "Client IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_internal_ip", - "label": "Internal IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_port", - "label": "Client Port", - "type": "int" - }, - { - "name": "common_l4_protocol", - "label": "L4 Protocol", - "type": "string" - }, - { - "name": "common_address_type", - "label": "Address Type", - "type": "int", - "doc": "{\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "label": "Server IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_server_port", - "label": "Server Port", - "type": "int", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_external_ip", - "label": "External IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_action", - "label": "Action", - "type": "int", - "doc": "{\"allow_query\":\"true\",\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"128\",\"value\":\"Allow\"}]}" - }, - { - "name": "common_direction", - "label": "Direction", - "type": "int", - "doc": "{\"data\":[{\"code\":\"69\",\"value\":\"outbound\"},{\"code\":\"73\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "label": "Sled IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "label": "Client Location", - "type": "string" - }, - { - "name": "common_client_asn", - "label": "Client ASN", - "type": "string" - }, - { - "name": "common_server_location", - "label": "Server Location", - "type": "string" - }, - { - "name": "common_server_asn", - "label": "Server ASN", - "type": "string" - }, - { - "name": "common_sessions", - "label": "Sessions", - "type": "long" - }, - { - "name": "common_c2s_pkt_num", - "label": "Packets Sent", - "type": "long" - }, - { - "name": "common_s2c_pkt_num", - "label": "Packets Received", - "type": "long" - }, - { - "name": "common_c2s_byte_num", - "label": "Bytes Sent", - "type": "long" - }, - { - "name": "common_s2c_byte_num", - "label": "Bytes Received", - "type": "long" - }, - { - "name": "common_service", - "label": "Service", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "label": "Schema Type", - "type": "string", - "doc": "{\"data\":[{\"code\":\"BASE\",\"value\":\"BASE\"},{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"MAIL\",\"value\":\"MAIL\"},{\"code\":\"DNS\",\"value\":\"DNS\"},{\"code\":\"SSL\",\"value\":\"SSL\"},{\"code\":\"FTP\",\"value\":\"FTP\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_user_tags", - "label": "User Tags", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "label": "Sub Action", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_user_region", - "label": "User Region", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "label": "Device ID", - "type": "string" - }, - { - "name": "common_link_id", - "label": "Link ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "label": "ISP", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_device_tag", - "label": "Device Tag", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_data_center", - "label": "Data Center", - "type": "string" - }, - { - "name": "common_encapsulation", - "label": "Encapsulation", - "type": "int", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}]}" - }, - { - "name": "common_app_label", - "label": "Application Label", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_protocol_label", - "label": "Protocol Label", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_id", - "label": "Application ID", - "type": "int" - }, - { - "name": "common_app_surrogate_id", - "label": "Surrogate ID", - "type": "int" - }, - { - "name": "common_l7_protocol", - "label": "L7 Protocol", - "type": "string" - }, - { - "name": "common_start_time", - "label": "Start Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"visibility\":\"hidden\"}" - }, - { - "name": "common_end_time", - "label": "End Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"visibility\":\"hidden\"}" - }, - { - "name": "common_establish_latency_ms", - "type": "int", - "label": "Establish Latency(ms)", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_con_duration_ms", - "label": "Duration(ms)", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_dir", - "label": "Stream Direction", - "type": "int", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "label": "Address List", - "type": "string", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_has_dup_traffic", - "label": "Duplication Traffic", - "type": "int", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "label": "Stream Error", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_trace_id", - "label": "Session ID", - "type": "long", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_ipfrag_num", - "label": "Fragmentation Packets(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_ipfrag_num", - "label": "Fragmentation Packets(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_tcp_lostlen", - "label": "Sequence Gap Loss(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_tcp_lostlen", - "label": "Sequence Gap Loss(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_tcp_unorder_num", - "label": "Unorder Packets(c2s)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_tcp_unorder_num", - "label": "Unorder Packets(s2c)", - "type": "long", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_tcp_client_isn", - "label": "TCP Client ISN", - "type": "long", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_tcp_server_isn", - "label": "TCP Server ISN", - "type": "long", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_first_ttl", - "label": "First TTL", - "type": "int", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_processing_time", - "label": "Processing Time", - "type": "long", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "nic_name", - "label": "Nic Name", - "type": "string" - }, - { - "name": "origin_source_mac", - "label": "Origin Source Mac", - "type": "string" - }, - { - "name": "origin_dest_mac", - "label": "Origin Dest Mac", - "type": "string" - }, - { - "name": "packet_url", - "label": "Packet URL", - "type": "string" - }, - { - "name": "pcap_storage_task_id", - "label": "Task ID", - "type": "int" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/tables_cluster.avsc b/config/avro/clickhouse/tables_cluster.avsc deleted file mode 100644 index 4765d85f..00000000 --- a/config/avro/clickhouse/tables_cluster.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace": "system", - "type": "record", - "name": "tables_cluster", - "fields": [ - { - "name": "database", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/clickhouse/voip_record_log.avsc b/config/avro/clickhouse/voip_record_log.avsc deleted file mode 100644 index 59c8c611..00000000 --- a/config/avro/clickhouse/voip_record_log.avsc +++ /dev/null @@ -1,455 +0,0 @@ -{ - "type": "record", - "name": "voip_record_log", - "namespace": "tsg_galaxy_v3", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"functions\":{\"aggregation\":[{\"name\":\"COUNT\",\"function\":\"count(expr)\"},{\"name\":\"COUNT_DISTINCT\",\"function\":\"count(distinct expr)\"},{\"name\":\"AVG\",\"function\":\"avg(expr)\"},{\"name\":\"SUM\",\"function\":\"sum(expr)\"},{\"name\":\"MAX\",\"function\":\"max(expr)\"},{\"name\":\"MIN\",\"function\":\"min(expr)\"}],\"operator\":[{\"name\":\"=\",\"function\":\"expr = value\"},{\"name\":\"!=\",\"function\":\"expr != value\"},{\"name\":\">\",\"function\":\"expr > value\"},{\"name\":\"<\",\"function\":\"expr < value\"},{\"name\":\">=\",\"function\":\"expr >= value\"},{\"name\":\"<=\",\"function\":\"expr <= value\"},{\"name\":\"in\",\"function\":\"expr in (values)\"},{\"name\":\"not in\",\"function\":\"expr not in (values)\"},{\"name\":\"like\",\"function\":\"expr like value\"},{\"name\":\"not like\",\"function\":\"expr not like value\"},{\"name\":\"not empty\",\"function\":\"notEmpty(expr)\"},{\"name\":\"empty\",\"function\":\"empty(expr)\"}]},\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_sled_ip\",\"common_device_id\",\"common_client_location\",\"common_server_location\",\"common_subscriber_id\",\"common_client_port\",\"common_server_port\",\"common_schema_type\",\"common_l4_protocol\",\"common_l7_protocol\",\"common_data_center\",\"common_client_asn\",\"common_server_asn\",\"common_start_time\",\"common_end_time\",\"sip_call_id\",\"sip_from\",\"sip_to\",\"sip_user_agent\",\"sip_server\",\"sip_duration\",\"sip_bye\",\"rtp_payload_type_c2s\",\"rtp_payload_type_s2c\",\"rtp_from_direction\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_subscriber_id\",\"common_sled_ip\",\"common_device_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_sessions\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"sip_call_id\",\"sip_from\",\"sip_to\",\"sip_user_agent\",\"sip_server\",\"sip_duration\"],\"filters\":[\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_internal_ip\",\"common_external_ip\",\"common_client_port\",\"common_server_port\",\"common_client_location\",\"common_server_location\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_l4_protocol\",\"common_l7_protocol\",\"common_stream_dir\",\"common_direction\",\"common_data_center\",\"common_sled_ip\",\"common_device_id\",\"common_schema_type\",\"common_client_asn\",\"common_server_asn\",\"common_start_time\",\"common_end_time\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"sip_call_id\",\"sip_from\",\"sip_to\",\"sip_user_agent\",\"sip_server\",\"sip_duration\",\"sip_bye\",\"rtp_payload_type_c2s\",\"rtp_payload_type_s2c\",\"rtp_from_direction\"],\"references\":{\"aggregation\":[{\"type\":\"int\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"long\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"float\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"double\",\"functions\":\"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"},{\"type\":\"string\",\"functions\":\"COUNT,COUNT_DISTINCT\"},{\"type\":\"date\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"},{\"type\":\"timestamp\",\"functions\":\"COUNT,COUNT_DISTINCT,MAX,MIN\"}],\"operator\":[{\"type\":\"int\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"long\",\"functions\":\"=,!=,>,<,>=,<=,in,not in\"},{\"type\":\"float\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"double\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"string\",\"functions\":\"=,!=,in,not in,like,not like,not empty,empty\"},{\"type\":\"date\",\"functions\":\"=,!=,>,<,>=,<=\"},{\"type\":\"timestamp\",\"functions\":\"=,!=,>,<,>=,<=\"}]},\"details\":{\"general\":[\"common_recv_time\",\"common_log_id\",\"common_stream_trace_id\",\"common_direction\",\"common_stream_dir\",\"common_start_time\",\"common_end_time\",\"common_con_duration_ms\",\"common_establish_latency_ms\",\"common_processing_time\"],\"action\":[\"common_action\",\"common_sub_action\",\"common_policy_id\",\"common_user_tags\",\"common_user_region\"],\"source\":[\"common_client_ip\",\"common_internal_ip\",\"common_client_port\",\"common_client_location\",\"common_client_asn\",\"common_entrance_id\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_data_center\",\"common_sled_ip\",\"common_tcp_client_isn\",\"common_subscriber_id\"],\"destination\":[\"common_server_ip\",\"common_external_ip\",\"common_server_port\",\"common_server_location\",\"common_server_asn\",\"common_tcp_server_isn\"],\"application\":[\"common_app_id\",\"common_app_label\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_protocol_label\",\"common_service\",\"common_l4_protocol\"],\"data\":[\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_first_ttl\"]}},\"schema_type\":{\"SIP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"sip_call_id\",\"sip_from\",\"sip_to\",\"sip_user_agent\",\"sip_server\",\"sip_from_sdp_connect_ip\",\"sip_from_sdp_media_port\",\"sip_from_sdp_media_type\",\"sip_from_sdp_content\",\"sip_to_sdp_connect_ip\",\"sip_to_sdp_media_port\",\"sip_to_sdp_media_type\",\"sip_to_sdp_content\",\"sip_duration\",\"sip_bye\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"sip_from\",\"sip_to\",\"sip_call_id\",\"common_server_ip\",\"common_server_port\"]},\"RTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"rtp_payload_type_c2s\",\"rtp_payload_type_s2c\",\"rtp_pcap_dir_c2s\",\"rtp_pcap_dir_s2c\",\"rtp_from_direction\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"rtp_pcap_dir_c2s\",\"rtp_pcap_dir_s2c\",\"rtp_from_direction\"]},\"VoIP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_internal_ip\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_external_ip\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_device_tag\",\"common_data_center\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_app_id\",\"common_app_surrogate_id\",\"common_l7_protocol\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"common_c2s_ipfrag_num\",\"common_s2c_ipfrag_num\",\"common_c2s_tcp_lostlen\",\"common_s2c_tcp_lostlen\",\"common_c2s_tcp_unorder_num\",\"common_s2c_tcp_unorder_num\",\"common_tcp_client_isn\",\"common_tcp_server_isn\",\"common_first_ttl\",\"common_processing_time\",\"sip_call_id\",\"sip_from\",\"sip_to\",\"sip_user_agent\",\"sip_server\",\"sip_from_sdp_connect_ip\",\"sip_from_sdp_media_port\",\"sip_from_sdp_media_type\",\"sip_from_sdp_content\",\"sip_to_sdp_connect_ip\",\"sip_to_sdp_media_port\",\"sip_to_sdp_media_type\",\"sip_to_sdp_content\",\"sip_duration\",\"sip_bye\",\"rtp_payload_type_c2s\",\"rtp_payload_type_s2c\",\"rtp_pcap_dir_c2s\",\"rtp_pcap_dir_s2c\",\"rtp_from_direction\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"sip_from\",\"sip_to\",\"sip_call_id\",\"common_server_ip\",\"common_server_port\",\"rtp_pcap_dir_c2s\",\"rtp_pcap_dir_s2c\",\"rtp_from_direction\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"sip_from\",\"sip_to\",\"sip_call_id\",\"common_server_ip\",\"common_server_port\",\"rtp_pcap_dir_c2s\",\"rtp_pcap_dir_s2c\",\"rtp_from_direction\"]}", - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"}}", - "type": "long" - }, - { - "name": "common_log_id", - "label": "Log ID", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}", - "type": "long" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "doc": "{\"visibility\":\"hidden\"}", - "type": "long" - }, - { - "name": "common_subscriber_id", - "label": "Subscriber ID", - "doc": "{\"allow_query\":\"true\"}", - "type": "string" - }, - { - "name": "common_client_ip", - "label": "Client IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_asn,radius_match\",\"appendTo\":\"common_client_asn,common_subscriber_id\"}}", - "type": "string" - }, - { - "name": "common_internal_ip", - "label": "Internal IP", - "doc": "{\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"if\",\"param\":\"$.common_direction=69,$.common_client_ip,$.common_server_ip\"},\"allow_query\":\"true\"}", - "type": "string" - }, - { - "name": "common_client_port", - "label": "Client Port", - "doc": "{\"allow_query\":\"true\"}", - "type": "int" - }, - { - "name": "common_l4_protocol", - "label": "L4 Protocol", - "type": "string" - }, - { - "name": "common_address_type", - "label": "Address Type", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}", - "type": "int" - }, - { - "name": "common_server_ip", - "label": "Server IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_asn\",\"appendTo\":\"common_server_asn\"}}", - "type": "string" - }, - { - "name": "common_server_port", - "label": "Server Port", - "doc": "{\"allow_query\":\"true\"}", - "type": "int" - }, - { - "name": "common_external_ip", - "label": "External IP", - "doc": "{\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"if\",\"param\":\"$.common_direction=73,$.common_client_ip,$.common_server_ip\"},\"allow_query\":\"true\"}", - "type": "string" - }, - { - "name": "common_action", - "label": "Action", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"128\",\"value\":\"Allow\"}]}", - "type": "int" - }, - { - "name": "common_direction", - "label": "Direction", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"69\",\"value\":\"outbound\"},{\"code\":\"73\",\"value\":\"inbound\"}]}", - "type": "int" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "doc": "{\"visibility\":\"disabled\"}", - "type": "int" - }, - { - "name": "common_sled_ip", - "label": "Sled IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}", - "type": "string" - }, - { - "name": "common_client_location", - "label": "Client Location", - "type": "string" - }, - { - "name": "common_client_asn", - "label": "Client ASN", - "type": "string" - }, - { - "name": "common_server_location", - "label": "Server Location", - "type": "string" - }, - { - "name": "common_server_asn", - "label": "Server ASN", - "type": "string" - }, - { - "name": "common_sessions", - "label": "Sessions", - "type": "long" - }, - { - "name": "common_c2s_pkt_num", - "label": "Packets Sent", - "type": "long" - }, - { - "name": "common_s2c_pkt_num", - "label": "Packets Received", - "type": "long" - }, - { - "name": "common_c2s_byte_num", - "label": "Bytes Sent", - "type": "long" - }, - { - "name": "common_s2c_byte_num", - "label": "Bytes Received", - "type": "long" - }, - { - "name": "common_service", - "label": "Service", - "doc": "{\"visibility\":\"disabled\"}", - "type": "int" - }, - { - "name": "common_schema_type", - "label": "Schema Type", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"VoIP\",\"value\":\"VoIP\"},{\"code\":\"SIP\",\"value\":\"SIP\"},{\"code\":\"RTP\",\"value\":\"RTP\"}],\"allow_query\":\"true\"}", - "type": "string" - }, - { - "name": "common_user_tags", - "label": "User Tags", - "doc": "{\"visibility\":\"disabled\"}", - "type": "string" - }, - { - "name": "common_sub_action", - "label": "Sub Action", - "type": "string" - }, - { - "name": "common_user_region", - "label": "User Region", - "doc": "{\"visibility\":\"hidden\"}", - "type": "string" - }, - { - "name": "common_device_id", - "label": "Device ID", - "type": "string" - }, - { - "name": "common_link_id", - "label": "Link ID", - "doc": "{\"visibility\":\"disabled\"}", - "type": "int" - }, - { - "name": "common_isp", - "label": "ISP", - "doc": "{\"visibility\":\"disabled\"}", - "type": "string" - }, - { - "name": "common_device_tag", - "label": "Device Tag", - "doc": "{\"visibility\":\"hidden\"}", - "type": "string" - }, - { - "name": "common_data_center", - "label": "Data Center", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"Nur-sultan\",\"value\":\"Nur-sultan\"},{\"code\":\"Aktau\",\"value\":\"Aktau\"},{\"code\":\"Aktubinsk\",\"value\":\"Aktubinsk\"},{\"code\":\"Almaty\",\"value\":\"Almaty\"},{\"code\":\"Atyrau\",\"value\":\"Atyrau\"},{\"code\":\"Karaganda\",\"value\":\"Karaganda\"},{\"code\":\"Kokshetau\",\"value\":\"Kokshetau\"},{\"code\":\"Kostanay\",\"value\":\"Kostanay\"},{\"code\":\"Kyzylorda\",\"value\":\"Kyzylorda\"},{\"code\":\"Pavlodar\",\"value\":\"Pavlodar\"},{\"code\":\"Petropavl\",\"value\":\"Petropavl\"},{\"code\":\"Semey\",\"value\":\"Semey\"},{\"code\":\"Shymkent\",\"value\":\"Shymkent\"},{\"code\":\"Taldykurgan\",\"value\":\"Taldykurgan\"},{\"code\":\"Taraz\",\"value\":\"Taraz\"},{\"code\":\"Uralsk\",\"value\":\"Uralsk\"},{\"code\":\"Ust-Kamenogorsk\",\"value\":\"Ust-Kamenogorsk\"},{\"code\":\"Zhezkazgan\",\"value\":\"Zhezkazgan\"}],\"allow_query\":\"true\"}", - "type": "string" - }, - { - "name": "common_encapsulation", - "label": "Encapsulation", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}", - "type": "int" - }, - { - "name": "common_app_label", - "label": "Application Label", - "type": "string" - }, - { - "name": "common_protocol_label", - "label": "Protocol Label", - "type": "string" - }, - { - "name": "common_app_id", - "label": "Application ID", - "type": "int" - }, - { - "name": "common_app_surrogate_id", - "label": "Surrogate ID", - "type": "int" - }, - { - "name": "common_l7_protocol", - "label": "L7 Protocol", - "type": "string" - }, - { - "name": "common_start_time", - "label": "Start Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}", - "type": "long" - }, - { - "name": "common_end_time", - "label": "End Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"get_value\",\"appendTo\":\"common_recv_time\"}}", - "type": "long" - }, - { - "name": "common_establish_latency_ms", - "label": "Establish Latency(ms)", - "type": "int" - }, - { - "name": "common_con_duration_ms", - "label": "Duration(ms)", - "type": "int" - }, - { - "name": "common_stream_dir", - "label": "Stream Direction", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}],\"allow_query\":\"true\"}", - "type": "int" - }, - { - "name": "common_address_list", - "label": "Address List", - "doc": "{\"visibility\":\"disabled\"}", - "type": "string" - }, - { - "name": "common_has_dup_traffic", - "label": "Duplication Traffic", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}", - "type": "int" - }, - { - "name": "common_stream_error", - "label": "Stream Error", - "doc": "{\"visibility\":\"hidden\"}", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "label": "Session ID", - "doc": "{\"allow_query\":\"true\"}", - "type": "long" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "doc": "{\"visibility\":\"hidden\"}", - "type": "string" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "doc": "{\"visibility\":\"hidden\"}", - "type": "string" - }, - { - "name": "common_c2s_ipfrag_num", - "label": "Fragmentation Packets(c2s)", - "type": "long" - }, - { - "name": "common_s2c_ipfrag_num", - "label": "Fragmentation Packets(s2c)", - "type": "long" - }, - { - "name": "common_c2s_tcp_lostlen", - "label": "Sequence Gap Loss(c2s)", - "type": "long" - }, - { - "name": "common_s2c_tcp_lostlen", - "label": "Sequence Gap Loss(s2c)", - "type": "long" - }, - { - "name": "common_c2s_tcp_unorder_num", - "label": "Unorder Packets(c2s)", - "type": "long" - }, - { - "name": "common_s2c_tcp_unorder_num", - "label": "Unorder Packets(s2c)", - "type": "long" - }, - { - "name": "common_tcp_client_isn", - "label": "TCP Client ISN", - "doc": "{\"allow_query\":\"true\"}", - "type": "long" - }, - { - "name": "common_tcp_server_isn", - "label": "TCP Server ISN", - "doc": "{\"allow_query\":\"true\"}", - "type": "long" - }, - { - "name": "common_first_ttl", - "label": "First TTL", - "doc": "{\"visibility\":\"hidden\"}", - "type": "int" - }, - { - "name": "common_processing_time", - "label": "Processing Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}", - "type": "long" - }, - { - "name": "sip_from", - "label": "SIP.From", - "type": "string" - }, - { - "name": "sip_to", - "label": "SIP.To", - "type": "string" - }, - { - "name": "sip_user_agent", - "label": "SIP.User-Agent", - "type": "string" - }, - { - "name": "sip_server", - "label": "SIP.Server", - "type": "string" - }, - { - "name": "sip_from_sdp_connect_ip", - "label": "SIP.From Connect IP", - "type": "string" - }, - { - "name": "sip_from_sdp_media_port", - "label": "SIP.From Media Port", - "type": "int" - }, - { - "name": "sip_from_sdp_media_type", - "label": "SIP.From Media Type", - "type": "string" - }, - { - "name": "sip_from_sdp_content", - "label": "SIP.From SDP Content", - "type": "string" - }, - { - "name": "sip_to_sdp_connect_ip", - "label": "SIP.To Connect IP", - "type": "string" - }, - { - "name": "sip_to_sdp_media_port", - "label": "SIP.To Media Port", - "type": "int" - }, - { - "name": "sip_to_sdp_media_type", - "label": "SIP.To Media Type", - "type": "string" - }, - { - "name": "sip_to_sdp_content", - "label": "SIP.To SDP Content", - "type": "string" - }, - { - "name": "sip_duration", - "label": "SIP.Duration", - "type": "int" - }, - { - "name": "sip_bye", - "label": "SIP.Bye", - "type": "string" - }, - { - "name": "rtp_payload_type_c2s", - "label": "RTP.Payload(c2s)", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"PCMU\"},{\"code\":\"1\",\"value\":\"1016\"},{\"code\":\"2\",\"value\":\"G721\"},{\"code\":\"3\",\"value\":\"GSM\"},{\"code\":\"4\",\"value\":\"G723\"},{\"code\":\"5\",\"value\":\"DVI4_8000\"},{\"code\":\"6\",\"value\":\"DVI4_16000\"},{\"code\":\"7\",\"value\":\"LPC\"},{\"code\":\"8\",\"value\":\"PCMA\"},{\"code\":\"9\",\"value\":\"G722\"},{\"code\":\"10\",\"value\":\"L16_STEREO\"},{\"code\":\"11\",\"value\":\"L16_MONO\"},{\"code\":\"12\",\"value\":\"QCELP\"},{\"code\":\"13\",\"value\":\"CN\"},{\"code\":\"14\",\"value\":\"MPA\"},{\"code\":\"15\",\"value\":\"G728\"},{\"code\":\"16\",\"value\":\"DVI4_11025\"},{\"code\":\"17\",\"value\":\"DVI4_22050\"},{\"code\":\"18\",\"value\":\"G729\"},{\"code\":\"19\",\"value\":\"CN_OLD\"},{\"code\":\"25\",\"value\":\"CELB\"},{\"code\":\"26\",\"value\":\"JPEG\"},{\"code\":\"28\",\"value\":\"NV\"},{\"code\":\"31\",\"value\":\"H261\"},{\"code\":\"32\",\"value\":\"MPV\"},{\"code\":\"33\",\"value\":\"MP2T\"},{\"code\":\"34\",\"value\":\"H263\"}]}", - "type": "int" - }, - { - "name": "rtp_payload_type_s2c", - "label": "RTP.Payload(s2c)", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"PCMU\"},{\"code\":\"1\",\"value\":\"1016\"},{\"code\":\"2\",\"value\":\"G721\"},{\"code\":\"3\",\"value\":\"GSM\"},{\"code\":\"4\",\"value\":\"G723\"},{\"code\":\"5\",\"value\":\"DVI4_8000\"},{\"code\":\"6\",\"value\":\"DVI4_16000\"},{\"code\":\"7\",\"value\":\"LPC\"},{\"code\":\"8\",\"value\":\"PCMA\"},{\"code\":\"9\",\"value\":\"G722\"},{\"code\":\"10\",\"value\":\"L16_STEREO\"},{\"code\":\"11\",\"value\":\"L16_MONO\"},{\"code\":\"12\",\"value\":\"QCELP\"},{\"code\":\"13\",\"value\":\"CN\"},{\"code\":\"14\",\"value\":\"MPA\"},{\"code\":\"15\",\"value\":\"G728\"},{\"code\":\"16\",\"value\":\"DVI4_11025\"},{\"code\":\"17\",\"value\":\"DVI4_22050\"},{\"code\":\"18\",\"value\":\"G729\"},{\"code\":\"19\",\"value\":\"CN_OLD\"},{\"code\":\"25\",\"value\":\"CELB\"},{\"code\":\"26\",\"value\":\"JPEG\"},{\"code\":\"28\",\"value\":\"NV\"},{\"code\":\"31\",\"value\":\"H261\"},{\"code\":\"32\",\"value\":\"MPV\"},{\"code\":\"33\",\"value\":\"MP2T\"},{\"code\":\"34\",\"value\":\"H263\"}]}", - "type": "int" - }, - { - "name": "rtp_pcap_dir_c2s", - "label": "RTP.PCAP(c2s)", - "doc": "{\"constraints\":{\"type\":\"file\"}}", - "type": "string" - }, - { - "name": "rtp_pcap_dir_s2c", - "label": "RTP.PCAP(s2c)", - "doc": "{\"constraints\":{\"type\":\"file\"}}", - "type": "string" - }, - { - "name": "rtp_from_direction", - "label": "Direction", - "doc": "{\"constraints\":{\"operator_functions\":\"=,!=\"},\"data\":[{\"code\":\"0\",\"value\":\"unknown\"},{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"}]}", - "type": "int" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/proxy_event_hits_log.avsc b/config/avro/druid/proxy_event_hits_log.avsc deleted file mode 100644 index f6f700c4..00000000 --- a/config/avro/druid/proxy_event_hits_log.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "proxy_event_hits_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "action", - "type": "long" - }, - { - "name": "isp", - "type": "string" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "hits", - "type": "long" - }, - { - "name": "policy_id", - "type": "long" - }, - { - "name": "sub_action", - "type": "string" - }, - { - "name": "country", - "type": "string" - }, - { - "name": "location", - "type": "string" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "ip_object", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/security_event_hits_log.avsc b/config/avro/druid/security_event_hits_log.avsc deleted file mode 100644 index 740c124e..00000000 --- a/config/avro/druid/security_event_hits_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "security_event_hits_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "action", - "type": "long" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "hits", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "isp", - "type": "string" - }, - { - "name": "policy_id", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/source_country_ip_num_log.avsc b/config/avro/druid/source_country_ip_num_log.avsc deleted file mode 100644 index 908fa08b..00000000 --- a/config/avro/druid/source_country_ip_num_log.avsc +++ /dev/null @@ -1,20 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "source_country_ip_num_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "country", - "type": "string" - }, - { - "name": "ip_object", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/sys_storage_log.avsc b/config/avro/druid/sys_storage_log.avsc deleted file mode 100644 index 79d6be4b..00000000 --- a/config/avro/druid/sys_storage_log.avsc +++ /dev/null @@ -1,36 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "sys_storage_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "log_type", - "type": "string" - }, - { - "name": "data_center", - "type": "string" - }, - { - "name": "max_size", - "type": "long" - }, - { - "name": "used_size", - "type": "long" - }, - { - "name": "aggregate_size", - "type": "long" - }, - { - "name": "last_storage", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/top_client_ip_log.avsc b/config/avro/druid/top_client_ip_log.avsc deleted file mode 100644 index 8dd059db..00000000 --- a/config/avro/druid/top_client_ip_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_client_ip_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "source", - "type": "string" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "s2c_byte_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/top_domain_recommend_daily_log.avsc b/config/avro/druid/top_domain_recommend_daily_log.avsc deleted file mode 100644 index 76766b54..00000000 --- a/config/avro/druid/top_domain_recommend_daily_log.avsc +++ /dev/null @@ -1,32 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_domain_recommend_daily_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "domain", - "type": "string" - }, - { - "name": "ip_num", - "type": "long" - }, - { - "name": "order_num", - "type": "long" - }, - { - "name": "protocol_type", - "type": "string" - }, - { - "name": "session_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/top_external_host_log.avsc b/config/avro/druid/top_external_host_log.avsc deleted file mode 100644 index 700e7746..00000000 --- a/config/avro/druid/top_external_host_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_external_host_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "destination", - "type": "string" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/top_internal_host_log.avsc b/config/avro/druid/top_internal_host_log.avsc deleted file mode 100644 index 8ecf9530..00000000 --- a/config/avro/druid/top_internal_host_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_internal_host_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "source", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/top_server_ip_log.avsc b/config/avro/druid/top_server_ip_log.avsc deleted file mode 100644 index 543d8f71..00000000 --- a/config/avro/druid/top_server_ip_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_server_ip_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "destination", - "type": "string" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "s2c_byte_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/top_urls_log.avsc b/config/avro/druid/top_urls_log.avsc deleted file mode 100644 index 44b5dbd9..00000000 --- a/config/avro/druid/top_urls_log.avsc +++ /dev/null @@ -1,20 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_urls_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "url", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/top_user_log.avsc b/config/avro/druid/top_user_log.avsc deleted file mode 100644 index 1f8dc0f2..00000000 --- a/config/avro/druid/top_user_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_user_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "subscriber_id", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/top_website_domain_log.avsc b/config/avro/druid/top_website_domain_log.avsc deleted file mode 100644 index b6bfc7fa..00000000 --- a/config/avro/druid/top_website_domain_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_website_domain_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "domain", - "type": "string" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/top_website_urls_daily_log.avsc b/config/avro/druid/top_website_urls_daily_log.avsc deleted file mode 100644 index 5679d146..00000000 --- a/config/avro/druid/top_website_urls_daily_log.avsc +++ /dev/null @@ -1,28 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_website_urls_daily_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "url", - "type": "string" - }, - { - "name": "ip_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "content_length", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/traffic_metrics_log.avsc b/config/avro/druid/traffic_metrics_log.avsc deleted file mode 100644 index ae672996..00000000 --- a/config/avro/druid/traffic_metrics_log.avsc +++ /dev/null @@ -1,212 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "traffic_metrics_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "allow_conn_num", - "type": "long" - }, - { - "name": "allow_in_bytes", - "type": "long" - }, - { - "name": "allow_in_packets", - "type": "long" - }, - { - "name": "allow_out_bytes", - "type": "long" - }, - { - "name": "allow_out_packets", - "type": "long" - }, - { - "name": "close_conn_num", - "type": "long" - }, - { - "name": "default_conn_num", - "type": "long" - }, - { - "name": "default_in_bytes", - "type": "long" - }, - { - "name": "default_in_packets", - "type": "long" - }, - { - "name": "default_out_bytes", - "type": "long" - }, - { - "name": "default_out_packets", - "type": "long" - }, - { - "name": "deny_conn_num", - "type": "long" - }, - { - "name": "deny_in_bytes", - "type": "long" - }, - { - "name": "deny_in_packets", - "type": "long" - }, - { - "name": "deny_out_bytes", - "type": "long" - }, - { - "name": "deny_out_packets", - "type": "long" - }, - { - "name": "device_id", - "type": "string" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "intercept_conn_num", - "type": "long" - }, - { - "name": "intercept_in_bytes", - "type": "long" - }, - { - "name": "intercept_in_packets", - "type": "long" - }, - { - "name": "intercept_out_bytes", - "type": "long" - }, - { - "name": "intercept_out_packets", - "type": "long" - }, - { - "name": "established_conn_num", - "type": "long" - }, - { - "name": "monitor_conn_num", - "type": "long" - }, - { - "name": "monitor_in_bytes", - "type": "long" - }, - { - "name": "monitor_in_packets", - "type": "long" - }, - { - "name": "monitor_out_bytes", - "type": "long" - }, - { - "name": "monitor_out_packets", - "type": "long" - }, - { - "name": "new_conn_num", - "type": "long" - }, - { - "name": "total_in_bytes", - "type": "long" - }, - { - "name": "total_in_packets", - "type": "long" - }, - { - "name": "total_out_bytes", - "type": "long" - }, - { - "name": "total_out_packets", - "type": "long" - }, - { - "name": "alert_bytes", - "type": "long" - }, - { - "name": "hijk_bytes", - "type": "long" - }, - { - "name": "ins_bytes", - "type": "long" - }, - { - "name": "intcp_allow_num", - "type": "long" - }, - { - "name": "intcp_deny_num", - "type": "long" - }, - { - "name": "intcp_hijk_num", - "type": "long" - }, - { - "name": "intcp_ins_num", - "type": "long" - }, - { - "name": "intcp_mon_num", - "type": "long" - }, - { - "name": "intcp_rdirt_num", - "type": "long" - }, - { - "name": "intcp_repl_num", - "type": "long" - }, - { - "name": "maybe_pinning_num", - "type": "long" - }, - { - "name": "not_pinning_num", - "type": "long" - }, - { - "name": "pinning_num", - "type": "long" - }, - { - "name": "ad_cc_bytes", - "type": "long" - }, - { - "name": "ad_flood_bytes", - "type": "long" - }, - { - "name": "ad_reflection_bytes", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/traffic_protocol_stat_log.avsc b/config/avro/druid/traffic_protocol_stat_log.avsc deleted file mode 100644 index 8945ffa4..00000000 --- a/config/avro/druid/traffic_protocol_stat_log.avsc +++ /dev/null @@ -1,72 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "traffic_protocol_stat_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "protocol_id", - "type": "string" - }, - { - "name": "isp", - "type": "string" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "data_center", - "type": "string" - }, - { - "name": "sessions", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "c2s_ipfrag_num", - "type": "long" - }, - { - "name": "s2c_ipfrag_num", - "type": "long" - }, - { - "name": "c2s_tcp_lostlen", - "type": "long" - }, - { - "name": "s2c_tcp_lostlen", - "type": "long" - }, - { - "name": "c2s_tcp_unorder_num", - "type": "long" - }, - { - "name": "s2c_tcp_unorder_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/druid/traffic_summary_log.avsc b/config/avro/druid/traffic_summary_log.avsc deleted file mode 100644 index e361e88a..00000000 --- a/config/avro/druid/traffic_summary_log.avsc +++ /dev/null @@ -1,72 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "traffic_summary_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "isp", - "type": "string" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "data_center", - "type": "string" - }, - { - "name": "schema_type", - "type": "string" - }, - { - "name": "ip_object", - "type": "string" - }, - { - "name": "sessions", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "one_sided_connections", - "type": "long" - }, - { - "name": "uncategorized_bytes", - "type": "long" - }, - { - "name": "fragmentation_packets", - "type": "long" - }, - { - "name": "sequence_gap_loss", - "type": "long" - }, - { - "name": "unorder_packets", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/avro/etl/liveChart.avsc b/config/avro/etl/liveChart.avsc deleted file mode 100644 index 9f3b5cd6..00000000 --- a/config/avro/etl/liveChart.avsc +++ /dev/null @@ -1,10 +0,0 @@ -{ - "type": "record", - "name": "liveChart", - "task": "Protocol-Distribution", - "in": "CONNECTION-RECORD-COMPLETED-LOG", - "out": "TRAFFIC-PROTOCOL-STAT-LOG", - "doc": "{\"timestamp\":{\"name\":\"stat_time\",\"type\":\"Long\"},\"dimensions\":[{\"name\":\"protocol_id\",\"fieldName\":\"common_protocol_label\",\"type\":\"String\"},{\"name\":\"entrance_id\",\"fieldName\":\"common_entrance_id\",\"type\":\"String\"},{\"name\":\"isp\",\"fieldName\":\"common_isp\",\"type\":\"String\"},{\"name\":\"data_center\",\"fieldName\":\"common_data_center\",\"type\":\"String\"}],\"metrics\":[{\"function\":\"sum\",\"name\":\"sessions\",\"fieldName\":\"common_sessions\"},{\"function\":\"sum\",\"name\":\"c2s_byte_num\",\"fieldName\":\"common_c2s_byte_num\"},{\"function\":\"sum\",\"name\":\"s2c_byte_num\",\"fieldName\":\"common_s2c_byte_num\"},{\"function\":\"sum\",\"name\":\"c2s_pkt_num\",\"fieldName\":\"common_c2s_pkt_num\"},{\"function\":\"sum\",\"name\":\"s2c_pkt_num\",\"fieldName\":\"common_s2c_pkt_num\"},{\"function\":\"sum\",\"name\":\"c2s_ipfrag_num\",\"fieldName\":\"common_c2s_ipfrag_num\"},{\"function\":\"sum\",\"name\":\"s2c_ipfrag_num\",\"fieldName\":\"common_s2c_ipfrag_num\"},{\"function\":\"sum\",\"name\":\"c2s_tcp_lostlen\",\"fieldName\":\"common_c2s_tcp_lostlen\"},{\"function\":\"sum\",\"name\":\"s2c_tcp_lostlen\",\"fieldName\":\"common_s2c_tcp_lostlen\"},{\"function\":\"sum\",\"name\":\"c2s_tcp_unorder_num\",\"fieldName\":\"common_c2s_tcp_unorder_num\"},{\"function\":\"sum\",\"name\":\"s2c_tcp_unorder_num\",\"fieldName\":\"common_s2c_tcp_unorder_num\"},{\"function\":\"disCount\",\"name\":\"unique_sip_num\",\"fieldName\":\"common_server_ip\"},{\"function\":\"disCount\",\"name\":\"unique_cip_num\",\"fieldName\":\"common_client_ip\"}],\"filters\":{\"type\":\"and\",\"fields\":[{\"fieldName\":\"common_protocol_label\",\"type\":\"not\",\"values\":null}]},\"transforms\":[{\"fieldName\":\"common_app_label\",\"function\":\"alignment\",\"name\":\"common_app_label\",\"parameters\":\"0,/\"},{\"function\":\"combination\",\"name\":\"protocol_id\",\"fieldName\":\"common_protocol_label\",\"parameters\":\"common_app_label,/\"},{\"function\":\"hierarchy\",\"name\":\"protocol_id\",\"fieldName\":\"common_protocol_label\",\"parameters\":\"/\"}],\"action\":[{\"label\":\"Default\",\"metrics\":\"sessions,c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num,c2s_ipfrag_num,s2c_ipfrag_num,c2s_tcp_lostlen,s2c_tcp_lostlen,c2s_tcp_unorder_num,s2c_tcp_unorder_num\"}],\"granularity\":{\"type\":\"period\",\"period\":\"5M\"}}", - "fields": [ - ] -}
\ No newline at end of file diff --git a/config/log4j2-prod.xml b/config/log4j2-prod.xml deleted file mode 100644 index 53ce48c3..00000000 --- a/config/log4j2-prod.xml +++ /dev/null @@ -1,56 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<configuration> - - <!--日志打印相关参数配置--> - <Properties> - <!--每100M压缩日志文件--> - <property name="LOG_SIZE">100M</property> - <!--最多产生20个压缩文件--> - <property name="LOG_NUMS">20</property> - <!--日志打印等级--> - <property name="LOG_LEVEL">info</property> - <!--日志文件路径--> - <property name="LOG_PATH">logs</property> - <!--日志文件名称--> - <property name="LOG_FILE_NAME">galaxy-qgw-service</property> - <!--日志打印格式--> - <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ss}] [%p] [Thread:%t] %l %x - %m%n</property> - </Properties> - - <appenders> - <Console name="consoleSystemOutAppender" target="SYSTEM_OUT"> - <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/> - <PatternLayout pattern="${LOG_PATTERN}"/> - </Console> - - <RollingFile name="rollingFileAllAppender" - fileName="${LOG_PATH}/${LOG_FILE_NAME}.log" - filePattern="${LOG_PATH}/history/$${date:yyyy-MM-dd}/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz"> - <PatternLayout pattern="${LOG_PATTERN}"/> - <Policies> - <SizeBasedTriggeringPolicy size="${LOG_SIZE}"/> - <TimeBasedTriggeringPolicy interval="1" modulate="true"/> - </Policies> - <Filters> - <ThresholdFilter level="all" onMatch="ACCEPT" onMismatch="DENY"/> - </Filters> - <DefaultRolloverStrategy max="${LOG_NUMS}"> - <Delete basePath="${LOG_PATH}/history" maxDepth="1"> - <IfFileName glob="*.log.gz"> - <IfLastModified age="180d"> - <IfAny> - <IfAccumulatedFileSize exceeds="200 GB" /> - </IfAny> - </IfLastModified> - </IfFileName> - </Delete> - </DefaultRolloverStrategy> - </RollingFile> - </appenders> - <loggers> - <root level="${LOG_LEVEL}"> - <appender-ref ref="consoleSystemOutAppender"/> - <appender-ref ref="rollingFileAllAppender"/> - </root> - </loggers> -</configuration>
\ No newline at end of file diff --git a/config/log4j2-test.xml b/config/log4j2-test.xml deleted file mode 100644 index 201ad491..00000000 --- a/config/log4j2-test.xml +++ /dev/null @@ -1,56 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<configuration> - - <!--日志打印相关参数配置--> - <Properties> - <!--每50M压缩日志文件--> - <property name="LOG_SIZE">50M</property> - <!--最多产生10个压缩文件--> - <property name="LOG_NUMS">10</property> - <!--日志打印等级--> - <property name="LOG_LEVEL">info</property> - <!--日志文件路径--> - <property name="LOG_PATH">logs</property> - <!--日志文件名称--> - <property name="LOG_FILE_NAME">galaxy-qgw-service</property> - <!--日志打印格式--> - <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ss}] [%p] [Thread:%t] %l %x - %m%n</property> - </Properties> - - <appenders> - <Console name="consoleSystemOutAppender" target="SYSTEM_OUT"> - <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/> - <PatternLayout pattern="${LOG_PATTERN}"/> - </Console> - - <RollingFile name="rollingFileAllAppender" - fileName="${LOG_PATH}/${LOG_FILE_NAME}.log" - filePattern="${LOG_PATH}/history/$${date:yyyy-MM-dd}/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz"> - <PatternLayout pattern="${LOG_PATTERN}"/> - <Policies> - <SizeBasedTriggeringPolicy size="${LOG_SIZE}"/> - <TimeBasedTriggeringPolicy interval="1" modulate="true"/> - </Policies> - <Filters> - <ThresholdFilter level="all" onMatch="ACCEPT" onMismatch="DENY"/> - </Filters> - <DefaultRolloverStrategy max="${LOG_NUMS}"> - <Delete basePath="${LOG_PATH}/history" maxDepth="1"> - <IfFileName glob="*.log.gz"> - <IfLastModified age="90d"> - <IfAny> - <IfAccumulatedFileSize exceeds="200 GB" /> - </IfAny> - </IfLastModified> - </IfFileName> - </Delete> - </DefaultRolloverStrategy> - </RollingFile> - </appenders> - <loggers> - <root level="${LOG_LEVEL}"> - <appender-ref ref="consoleSystemOutAppender"/> - <appender-ref ref="rollingFileAllAppender"/> - </root> - </loggers> -</configuration>
\ No newline at end of file diff --git a/config/log4j2-dev.xml b/config/log4j2.xml index 5ec94943..5ec94943 100644 --- a/config/log4j2-dev.xml +++ b/config/log4j2.xml diff --git a/config/version.json b/config/version.json deleted file mode 100644 index 63888644..00000000 --- a/config/version.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "product": "186 Environment", - "version": "20.08", - "registered": "Geedge", - "updated":"2020-08-01 00:00:00", - "components": { - "oss": [ - { - "name": "zookeeper", - "version": "3.4.9", - "licenseType": "Apache License 2.0", - "description": "分布式应用程序协调服务" - }, - { - "name": "kafka", - "version": "2.11_1.0.0", - "licenseType": "Apache License 2.0", - "description": "消息队列" - }, - { - "name": "habse", - "version": "2.2.3", - "licenseType": "Apache License 2.0", - "description": "用于文件系统和存储Radius数据" - }, - { - "name": "flume", - "version": "1.9.0", - "licenseType": "Apache License 2.0", - "description": "日志补全传输" - }, - { - "name": "clickhouse", - "version": "19.17.4.11", - "licenseType": "Apache License 2.0", - "description": "原始日志数据库" - }, - { - "name": "druid", - "version": "0.16.0", - "licenseType": "Apache License 2.0", - "description": "分析实时数据并提供低延迟查询的OLAP应用程序" - }, - { - "name": "gohangout", - "version": "1.14.4.20200616", - "description": "动态获取原始日志表schema入库程序" - } - ], - "apps": [ - { - "name": "galaxy-qgw-service", - "version": "v3.2.200730", - "description": "数据平台对外统一查询网关" - }, - { - "name": "galaxy-report-service", - "version": "20.07.08", - "description": "自定义报表查询服务" - }, - { - "name": "galaxy-hos-service", - "version": "20.07.17", - "description": "对象存储服务" - }, - { - "name": "xxl-job-admin", - "version": "v1.0.2.20200506", - "description": "分布式任务调度平台" - }, - { - "name": "xxl-job", - "version": "v1.0.2.20200628", - "description": "分布式任务调度平台-执行器" - } - ], - "tasks": [ - { - "name": "flume", - "version": "flume-config-20.08", - "description": "原始日志补全、subscriber更新、Radius上下线功能" - }, - { - "name": "druid", - "version": "druid-config-20.08", - "description": "所有分析日志任务" - }, - { - "name": "gohangout", - "version": "gohangout-config-20.08", - "description": "原始日志入库、上下线日志入库" - } - ] - } -}
\ No newline at end of file diff --git a/docs/release/release-341.md b/docs/release/release-341.md index 942657d0..e02967a0 100644 --- a/docs/release/release-341.md +++ b/docs/release/release-341.md @@ -8,6 +8,7 @@ * 新增集成jar进行加密功能，docker部署可进行加密。 * 新增集成jar未变化情况下，MD5保持固定。 * 修复聚合查询分布式子查询与预期结果不一致(TSG-5634) + * 扩展Schema，通联支持输出FTP信息(TSG-5740)
\ No newline at end of file @@ -72,7 +72,7 @@ <docker.registry>192.168.40.153</docker.registry> <docker.registry.port>9080</docker.registry.port> <docker.image.prefix>tsg/galaxy</docker.image.prefix> - + <nacos.config.version>0.2.7</nacos.config.version> </properties> <dependencies> @@ -249,19 +249,16 @@ <artifactId>hutool-all</artifactId> <version>5.5.4</version> </dependency> - <dependency> <groupId>com.github.java-json-tools</groupId> <artifactId>json-schema-validator</artifactId> <version>${json.schema.version}</version> </dependency> - <dependency> <groupId>com.jayway.jsonpath</groupId> <artifactId>json-path</artifactId> <version>${json.path.version}</version> </dependency> - <dependency> <groupId>com.jfinal</groupId> <artifactId>activerecord</artifactId> @@ -278,8 +275,11 @@ </exclusion> </exclusions> </dependency> - - + <dependency> + <groupId>com.alibaba.boot</groupId> + <artifactId>nacos-config-spring-boot-starter</artifactId> + <version>${nacos.config.version}</version> + </dependency> </dependencies> <build> @@ -549,10 +549,6 @@ </plugin> - - - - <plugin> <groupId>com.spotify</groupId> <artifactId>docker-maven-plugin</artifactId> diff --git a/src/main/java/com/mesalab/calcite/CalciteMemoryUtils.java b/src/main/java/com/mesalab/calcite/CalciteMemoryUtils.java index f6ff64ca..5e56d20c 100644 --- a/src/main/java/com/mesalab/calcite/CalciteMemoryUtils.java +++ b/src/main/java/com/mesalab/calcite/CalciteMemoryUtils.java @@ -183,7 +183,7 @@ public class CalciteMemoryUtils { return dummyTable; } - private static final String intUnits = "Int8,Int16,Int32,Int64,UInt8,UInt16,UInt32,UInt64,bigint"; + private static final String intUnits = "Int8,Int16,Int32,Int64,UInt8,UInt16,UInt32,UInt64,bigint,int,long"; private static final String doubleUnits = "Float32,Float64,float,double"; } diff --git a/src/main/java/com/mesalab/common/base/DataTypeMapping.java b/src/main/java/com/mesalab/common/base/DataTypeMapping.java new file mode 100644 index 00000000..255ce8f6 --- /dev/null +++ b/src/main/java/com/mesalab/common/base/DataTypeMapping.java @@ -0,0 +1,18 @@ +package com.mesalab.common.base; + +/** + * TODO + * + * @Classname DataTypeMapping + * @Date 2021/3/17 11:29 上午 + * @Author wWei + */ +public class DataTypeMapping { + public static final String INT = "int"; + public static final String LONG = "long"; + public static final String FLOAT = "float"; + public static final String DOUBLE = "double"; + public static final String STRING = "string"; + public static final String DATE = "date"; + public static final String TIMESTAMP = "timestamp"; +} diff --git a/src/main/java/com/mesalab/common/configuration/ClickHouseConfiguration.java b/src/main/java/com/mesalab/common/configuration/ClickHouseConfiguration.java deleted file mode 100644 index fe684af4..00000000 --- a/src/main/java/com/mesalab/common/configuration/ClickHouseConfiguration.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.mesalab.common.configuration; - -import com.mesalab.qgw.model.api.ClickHouseHttpSource; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Slf4j -@Configuration -@EnableConfigurationProperties(ClickHouseProperties.class) -public class ClickHouseConfiguration { - - private final ClickHouseProperties clickhouseProperties; - - @Autowired - public ClickHouseConfiguration(ClickHouseProperties clickhouseProperties) { - this.clickhouseProperties = clickhouseProperties; - } - - @Bean(name="clickHouseHttpSource") - public ClickHouseHttpSource clickhouseHttpSource() { - ClickHouseHttpSource httpSource = new ClickHouseHttpSource(); - httpSource.setUrl(StringUtil.substringAfter(clickhouseProperties.getUrl(), "//")); - httpSource.setDbName(clickhouseProperties.getDbname()); - httpSource.setRealTimeAccount(clickhouseProperties.getRealTimeAccount()); - httpSource.setLongTermAccount(clickhouseProperties.getLongTermAccount()); - - return httpSource; - } - -} diff --git a/src/main/java/com/mesalab/common/configuration/ClickHouseProperties.java b/src/main/java/com/mesalab/common/configuration/ClickHouseProperties.java deleted file mode 100644 index 289d92b7..00000000 --- a/src/main/java/com/mesalab/common/configuration/ClickHouseProperties.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.mesalab.common.configuration; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; - -@Data -@ConfigurationProperties(prefix = "clickhouse") -public class ClickHouseProperties { - private String url; - private String driverClassName; - private String dbname; - - private DBAccount realTimeAccount = new DBAccount(); - - private DBAccount longTermAccount = new DBAccount(); - - @Data - public class DBAccount { - private String username; - private String userKey; - - } - -} diff --git a/src/main/java/com/mesalab/common/configuration/DruidIoConfiguration.java b/src/main/java/com/mesalab/common/configuration/DruidIoConfiguration.java deleted file mode 100644 index 03da75d5..00000000 --- a/src/main/java/com/mesalab/common/configuration/DruidIoConfiguration.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.mesalab.common.configuration; - -import com.mesalab.qgw.model.api.DruidIoHttpSource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Slf4j -@Configuration -@EnableConfigurationProperties(DruidIoProperties.class) -public class DruidIoConfiguration { - - private final DruidIoProperties druidIoProperties; - - @Autowired - public DruidIoConfiguration(DruidIoProperties druidIoProperties) { - this.druidIoProperties = druidIoProperties; - } - - - @Bean(name="druidIoHttpSource") - public DruidIoHttpSource druidIoHttpSource() { - DruidIoHttpSource httpSource = new DruidIoHttpSource(); - httpSource.setUrl(druidIoProperties.getUrl()); - httpSource.setDbname(druidIoProperties.getDbname()); - return httpSource; - } - - - -} diff --git a/src/main/java/com/mesalab/common/configuration/DruidIoProperties.java b/src/main/java/com/mesalab/common/configuration/DruidIoProperties.java deleted file mode 100644 index ce996db7..00000000 --- a/src/main/java/com/mesalab/common/configuration/DruidIoProperties.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.common.configuration; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; - -@Data -@ConfigurationProperties(prefix = "druid") -public class DruidIoProperties { - private String url; - private String dbname; - -} diff --git a/src/main/java/com/mesalab/common/configuration/EngineConfiguration.java b/src/main/java/com/mesalab/common/configuration/EngineConfiguration.java deleted file mode 100644 index 39267ef2..00000000 --- a/src/main/java/com/mesalab/common/configuration/EngineConfiguration.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.mesalab.common.configuration; - -import com.mesalab.qgw.model.api.EngineConfigSource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Slf4j -@Configuration -@EnableConfigurationProperties(EngineProperties.class) -public class EngineConfiguration { - - private final EngineProperties engineProperties; - - @Autowired - public EngineConfiguration(EngineProperties engineProperties) { - this.engineProperties = engineProperties; - } - - @Bean(name="engineConfigSource") - public EngineConfigSource engineConfigSource() { - EngineConfigSource engineConfigSource = new EngineConfigSource(); - engineConfigSource.setMaxCacheNum(engineProperties.getMaxCacheNum()); - engineConfigSource.setDefaultResultNum(engineProperties.getDefaultResultNum()); - return engineConfigSource; - } - - - - - - - - - - - - - - - - - - -} diff --git a/src/main/java/com/mesalab/common/configuration/EngineProperties.java b/src/main/java/com/mesalab/common/configuration/EngineProperties.java deleted file mode 100644 index 11207653..00000000 --- a/src/main/java/com/mesalab/common/configuration/EngineProperties.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.common.configuration; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Data -@ConfigurationProperties(prefix = "engine") -public class EngineProperties { - private int maxCacheNum; - private int defaultResultNum; -} diff --git a/src/main/java/com/mesalab/common/configuration/HBaseConfiguration.java b/src/main/java/com/mesalab/common/configuration/HBaseConfiguration.java index a3c70161..32ed9f50 100644 --- a/src/main/java/com/mesalab/common/configuration/HBaseConfiguration.java +++ b/src/main/java/com/mesalab/common/configuration/HBaseConfiguration.java @@ -6,7 +6,6 @@ import com.mesalab.qgw.model.api.HBaseAPISource; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -14,30 +13,11 @@ import java.io.IOException; @Configuration -@EnableConfigurationProperties(HBaseProperties.class) public class HBaseConfiguration { private static final Log log = LogFactory.get(); - private final HBaseProperties hBaseProperties; - @Autowired - public HBaseConfiguration(HBaseProperties hBaseProperties) { - this.hBaseProperties = hBaseProperties; - } + private HBaseAPISource hBaseAPISource; - @Bean(name = "hBaseAPISource") - public HBaseAPISource hBaseHttpSource() { - HBaseAPISource apiSource = new HBaseAPISource(); - apiSource.setZookeeperQuorum(hBaseProperties.getZookeeperQuorum()); - apiSource.setZookeeperPropertyClientPort(hBaseProperties.getZookeeperPropertyClientPort()); - apiSource.setZookeeperParent(hBaseProperties.getZookeeperZnodeParent()); - apiSource.setClientIpcPoolSize(hBaseProperties.getClientIpcPoolSize()); - apiSource.setRpcTimeout(hBaseProperties.getRpcTimeout()); - apiSource.setDbName(hBaseProperties.getDbName()); - apiSource.setTableName(hBaseProperties.getTableName()); - apiSource.setColumnFamily(hBaseProperties.getColumnFamily()); - apiSource.setColumnName(hBaseProperties.getColumnName()); - return apiSource; - } @Bean(name = "hbaseConnection") public Connection getConnection() { @@ -46,11 +26,11 @@ public class HBaseConfiguration { try { if (conf == null) { conf = org.apache.hadoop.hbase.HBaseConfiguration.create(); - conf.set("hbase.zookeeper.quorum", hBaseProperties.getZookeeperQuorum()); - conf.set("hbase.zookeeper.property.clientPort", hBaseProperties.getZookeeperPropertyClientPort()); - conf.set("hbase.zookeeper.znode.parent", hBaseProperties.getZookeeperZnodeParent()); - conf.set("hbase.rpc.timeout", hBaseProperties.getRpcTimeout()); - conf.set("hbase.client.ipc.pool.size", hBaseProperties.getClientIpcPoolSize()); + conf.set("hbase.zookeeper.quorum", hBaseAPISource.getZookeeperQuorum()); + conf.set("hbase.zookeeper.property.clientPort", hBaseAPISource.getZookeeperPropertyClientPort()); + conf.set("hbase.zookeeper.znode.parent", hBaseAPISource.getZookeeperZnodeParent()); + conf.set("hbase.rpc.timeout", hBaseAPISource.getRpcTimeout()); + conf.set("hbase.client.ipc.pool.size", hBaseAPISource.getClientIpcPoolSize()); conf.set("hbase.client.ipc.pool.type", "ThreadLocalPool"); conf.set("zookeeper.recovery.retry.intervalmill", "200"); conf.set("zookeeper.recovery.retry", "3"); diff --git a/src/main/java/com/mesalab/common/configuration/HBaseProperties.java b/src/main/java/com/mesalab/common/configuration/HBaseProperties.java deleted file mode 100644 index 9c6cb675..00000000 --- a/src/main/java/com/mesalab/common/configuration/HBaseProperties.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.mesalab.common.configuration; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; - -@Data -@ConfigurationProperties(prefix = "hbase") -public class HBaseProperties { - private String zookeeperQuorum; - private String zookeeperPropertyClientPort; - private String zookeeperZnodeParent; - private String dbName; - private String tableName; - private String columnFamily; - private String columnName; - private String clientIpcPoolSize; - private String rpcTimeout; -} diff --git a/src/main/java/com/mesalab/common/configuration/JobAdminConfiguration.java b/src/main/java/com/mesalab/common/configuration/JobAdminConfiguration.java deleted file mode 100644 index dbfe9e0b..00000000 --- a/src/main/java/com/mesalab/common/configuration/JobAdminConfiguration.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.mesalab.common.configuration; - -import com.mesalab.qgw.model.api.JobAdminHttpSource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Slf4j -@Configuration -@EnableConfigurationProperties(JobAdminProperties.class) -public class JobAdminConfiguration { - - private final JobAdminProperties jobAdminProperties; - - @Autowired - public JobAdminConfiguration(JobAdminProperties jobAdminProperties) { - this.jobAdminProperties = jobAdminProperties; - } - - @Bean(name = "jobAdminHttpSource") - public JobAdminHttpSource jobAdminHttpSource() { - JobAdminHttpSource httpSource = new JobAdminHttpSource(); - httpSource.setUrl(jobAdminProperties.getUrl()); - httpSource.setUserName(jobAdminProperties.getUserName()); - httpSource.setPassword(jobAdminProperties.getPassword()); - return httpSource; - } -}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/common/configuration/JobAdminProperties.java b/src/main/java/com/mesalab/common/configuration/JobAdminProperties.java deleted file mode 100644 index 98c20e9f..00000000 --- a/src/main/java/com/mesalab/common/configuration/JobAdminProperties.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.mesalab.common.configuration; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; - -@Data -@ConfigurationProperties(prefix = "xxl-job-admin") -public class JobAdminProperties { - - private String url; - - private String userName; - - private String password; - -} diff --git a/src/main/java/com/mesalab/common/configuration/ProjectAuthorProperties.java b/src/main/java/com/mesalab/common/configuration/ProjectAuthorProperties.java deleted file mode 100644 index 767e28d6..00000000 --- a/src/main/java/com/mesalab/common/configuration/ProjectAuthorProperties.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.mesalab.common.configuration; - -import lombok.Data; - -/** - * 项目作者联系方式 - * - * @author dazzlzy - * @date 2018/5/26 - */ -@Data -public class ProjectAuthorProperties { - - private String name; - - private String url; - - private String email; - -} diff --git a/src/main/java/com/mesalab/common/configuration/ProjectProperties.java b/src/main/java/com/mesalab/common/configuration/ProjectProperties.java deleted file mode 100644 index 84ac4ac3..00000000 --- a/src/main/java/com/mesalab/common/configuration/ProjectProperties.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.mesalab.common.configuration; - -import com.mesalab.common.enums.EnvironmentEnum; -import com.mesalab.common.enums.EnvironmentGroupEnum; -import lombok.Data; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Repository; - -import java.util.ArrayList; -import java.util.List; - -/** - * 项目配置 - * - * @author dazzlzy - * @date 2018/5/26 - */ -@Data -@Repository -@ConfigurationProperties("project") -public class ProjectProperties { - - /** - * 工程名 - */ - private String name; - - /** - * 版本 - */ - private String version; - - /** - * 工程描述 - */ - private String description; - - /** - * 项目组织标识 - */ - private String groupId; - - /** - * 项目标识 - */ - private String artifactId; - - /** - * 项目根目录 - */ - private String basedir; - - /** - * 核心项目包 - */ - private String corePackage; - - /** - * 业务项目包 - */ - private String servicePackage; - - /** - * 当前环境值 - */ - private String[] env; - - /** - * 项目作者 - */ - private ProjectAuthorProperties author; - - /** - * 注入的spring环境上下文 - */ - private final Environment environment; - - @Autowired - public ProjectProperties(Environment environment) { - this.environment = environment; - this.env = environment.getActiveProfiles(); - } - - /** - * 是否是生产环境 - * 从运行环境中读取最后一个是否为生产环境 - * - * @return boolean 是否为生产环境 - */ - public boolean isProduct() { - List<String> runtimeEnvs = new ArrayList<>(); - for (String s : this.env) { - if (EnvironmentGroupEnum.isRuntime(s)) { - runtimeEnvs.add(s); - } - } - if (runtimeEnvs.size() == 0) { - return false; - } - //最后一个运行环境, 如果spring.profiles.active=dev, prod, mysql 则运行环境为dev, prod, 最后一个运行环境为prod，是生产环境 - String env = runtimeEnvs.get(runtimeEnvs.size() - 1); - return EnvironmentEnum.PROD.getName().equals(env); - } - -} diff --git a/src/main/java/com/mesalab/common/enums/MetadataTypeEnum.java b/src/main/java/com/mesalab/common/enums/MetadataTypeEnum.java new file mode 100644 index 00000000..fe626f1d --- /dev/null +++ b/src/main/java/com/mesalab/common/enums/MetadataTypeEnum.java @@ -0,0 +1,24 @@ +package com.mesalab.common.enums; + +import lombok.Getter; + +/** + * @Date: 2021-03-11 16:05 + * @Author : liuyongqiang + * @ClassName : MetadataTypeEnum + * @Description : 元数据类型枚举 + */ +@Getter +public enum MetadataTypeEnum { + + TABLES("tables"), + + FIELDS("fields"), + ; + + private String value; + + MetadataTypeEnum(String value) { + this.value = value; + } +} diff --git a/src/main/java/com/mesalab/common/nacos/NacosConfig.java b/src/main/java/com/mesalab/common/nacos/NacosConfig.java new file mode 100644 index 00000000..ae950ebd --- /dev/null +++ b/src/main/java/com/mesalab/common/nacos/NacosConfig.java @@ -0,0 +1,35 @@ +package com.mesalab.common.nacos; + +import com.alibaba.nacos.api.PropertyKeyConst; +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Properties; + +/** + * @Date: 2021-03-11 17:36 + * @Author : liuyongqiang + * @ClassName : NacosClientConfig + * @Description : NacosClientConfig + */ +@Data +@Component +public class NacosConfig { + + @Value("${nacos.config.server-addr}") + private String serverAddr; + @Value("${nacos.config.username}") + private String username; + @Value("${nacos.config.password}") + private String password; + + public Properties getProperties(String namespace) { + Properties properties = new Properties(); + properties.put(PropertyKeyConst.SERVER_ADDR, serverAddr); + properties.put(PropertyKeyConst.USERNAME, username); + properties.put(PropertyKeyConst.PASSWORD, password); + properties.put(PropertyKeyConst.NAMESPACE, namespace); + return properties; + } +} diff --git a/src/main/java/com/mesalab/common/nacos/NacosConst.java b/src/main/java/com/mesalab/common/nacos/NacosConst.java new file mode 100644 index 00000000..71909606 --- /dev/null +++ b/src/main/java/com/mesalab/common/nacos/NacosConst.java @@ -0,0 +1,40 @@ +package com.mesalab.common.nacos; + +/** + * @Date: 2021-03-17 09:58 + * @Author : liuyongqiang + * @ClassName : NacosConst + * @Description : NacosConst + */ +public class NacosConst { + + public static final String NACOS = "_nacos"; + + public static final String FIXED = "fixed-"; + + public static final String NAMESPACE_AVRO = "nacos-namespace-avro"; + + public static final String META_DATA_ID = "meta_data"; + + public static final String CLICKHOUSE_GROUP = "CLICKHOUSE_GROUP"; + + public static final String DRUID_GROUP = "DRUID_GROUP"; + + public static final String DEFAULT_GROUP = "DEFAULT_GROUP"; + + public static final String SNAPSHOT_TENANT = "snapshot-tenant"; + + public final static String NAMESPACE_GALAXY = "nacos-namespace-galaxy"; + + public static final String CK_FILTER_DATA_ID = "galaxy-qgw-ck-filter"; + + public static final String CK_QUERIES_TEMPLATE_DATA_ID = "galaxy-qgw-ck-queries-template"; + + public static final String DRUID_FILTER_DATA_ID = "galaxy-qgw-druid-filter"; + + public static final String DRUID_QUERIES_TEMPLATE_DATA_ID = "galaxy-qgw-druid-queries-template"; + + public static final String ENGINE_FILTER_DATA_ID = "galaxy-qgw-engine-filter"; + + public static final String ENGINE_QUERIES_TEMPLATE_DATA_ID = "galaxy-qgw-engine-queries-template"; +} diff --git a/src/main/java/com/mesalab/common/utils/SQLHelper.java b/src/main/java/com/mesalab/common/utils/SQLHelper.java index fd118b86..c06d9f4d 100644 --- a/src/main/java/com/mesalab/common/utils/SQLHelper.java +++ b/src/main/java/com/mesalab/common/utils/SQLHelper.java @@ -121,15 +121,15 @@ public class SQLHelper { return i < 0 ? tableName : tableName.substring(i + 1); } else { log.error("SQL Syntax Error: Get Table Name Error, SQL is:{}", sql); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error: Get Table Name Error."); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error: Get Table Name Error.", null); } } else { log.warn("Not support DML Parser"); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error, Not support DML Parser."); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error, Not support DML Parser.", null); } } catch (JSQLParserException e) { log.error("SQL Syntax Error: Get Table Name Error, SQL is:{}, Error is:{}", sql, e); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "SQL Syntax Error: ", ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), e.getCause()); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error.", e.getCause()); } } diff --git a/src/main/java/com/mesalab/common/utils/TableFinder.java b/src/main/java/com/mesalab/common/utils/TableFinder.java index 4ca55b79..aba6bd92 100644 --- a/src/main/java/com/mesalab/common/utils/TableFinder.java +++ b/src/main/java/com/mesalab/common/utils/TableFinder.java @@ -60,7 +60,7 @@ import net.sf.jsqlparser.statement.values.ValuesStatement; public class TableFinder implements SelectVisitor, FromItemVisitor, ExpressionVisitor, ItemsListVisitor, SelectItemVisitor, StatementVisitor { - private static MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("metadataService"); + private static MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("nacosMetadataService"); private static final String NOT_SUPPORTED_YET = "Not supported yet."; private boolean defaultAlias = false; diff --git a/src/main/java/com/mesalab/knowledge/common/config/ArangoConfig.java b/src/main/java/com/mesalab/knowledge/common/config/ArangoConfig.java index 9dd84d77..e6f310c1 100644 --- a/src/main/java/com/mesalab/knowledge/common/config/ArangoConfig.java +++ b/src/main/java/com/mesalab/knowledge/common/config/ArangoConfig.java @@ -1,8 +1,11 @@ package com.mesalab.knowledge.common.config; +import com.alibaba.nacos.api.config.ConfigType; +import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Component; /** * @description: for http @@ -10,28 +13,22 @@ import org.springframework.context.annotation.Configuration; * @create: 2020-07-07 **/ @Data -@Configuration +@Component +@NacosConfigurationProperties(prefix = "arango", dataId = "galaxy-qgw-service", type = ConfigType.YAML, autoRefreshed = true) public class ArangoConfig { - @Value("${arango.server}") private String server; - @Value("${arango.database}") private String database; - @Value("${arango.username}") private String username; - @Value("${arango.password}") private String password; - @Value("${arango.jwturl}") private String jwturl; - @Value("${arango.queryurl}") private String queryurl; - @Value("${arango.maxrows}") private String maxrows; private String jwt; diff --git a/src/main/java/com/mesalab/knowledge/common/utils/HttpConfig.java b/src/main/java/com/mesalab/knowledge/common/utils/HttpConfig.java index c7a8acc3..fc998921 100644 --- a/src/main/java/com/mesalab/knowledge/common/utils/HttpConfig.java +++ b/src/main/java/com/mesalab/knowledge/common/utils/HttpConfig.java @@ -1,7 +1,7 @@ package com.mesalab.knowledge.common.utils; +import com.alibaba.nacos.api.config.annotation.NacosValue; import lombok.Data; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Data @@ -10,55 +10,55 @@ public class HttpConfig { /** * 最大连接数400 */ - @Value("${http.pool.max.connection}") + @NacosValue(value = "${http.pool.max.connection}", autoRefreshed = true) private int maxConnectionNum; /** * 单路由最大连接数80 */ - @Value("${http.pool.max.per.route}") + @NacosValue(value = "${http.pool.max.per.route}", autoRefreshed = true) private int maxPerRoute; /** * 向服务端请求超时时间设置(单位:毫秒) */ - @Value("${http.pool.request.timeout}") + @NacosValue(value = "${http.pool.request.timeout}", autoRefreshed = true) private int serverRequestTimeOut; /** * 向服务端连接超时时间设置(单位:毫秒) */ - @Value("${http.pool.connect.timeout}") + @NacosValue(value = "${http.pool.connect.timeout}", autoRefreshed = true) private int serverConnectTimeOut; /** * 服务端响应超时时间设置(单位:毫秒) */ - @Value("${http.pool.response.timeout}") + @NacosValue(value = "${http.pool.response.timeout}", autoRefreshed = true) private int serverResponseTimeOut; /** * Druid查询请求响应时间 */ - @Value("${druid.socketTimeOut}") + @NacosValue(value = "${druid.socketTimeOut}", autoRefreshed = true) private int druidSocketTimeOut; /** * Arango查询请求响应时间 */ - @Value("${arango.socketTimeOut}") + @NacosValue(value = "${arango.socketTimeOut}", autoRefreshed = true) private int arangoSocketTimeOut; /** * ClickHouse长查询请求响应时间 */ - @Value("${clickhouse.longTermAccount.socketTimeOut}") + @NacosValue(value = "${clickhouse.longTermAccount.socketTimeOut}", autoRefreshed = true) private int ckLongTermAccountSocketTimeOut; /** * ClickHouse实时查询请求响应时间 */ - @Value("${clickhouse.realTimeAccount.socketTimeOut}") + @NacosValue(value = "${clickhouse.realTimeAccount.socketTimeOut}", autoRefreshed = true) private int ckRealTimeAccountSocketTimeOut; } diff --git a/src/main/java/com/mesalab/qgw/benchmark/Writer.java b/src/main/java/com/mesalab/qgw/benchmark/Writer.java index 489f15ba..39978c55 100644 --- a/src/main/java/com/mesalab/qgw/benchmark/Writer.java +++ b/src/main/java/com/mesalab/qgw/benchmark/Writer.java @@ -1,22 +1,25 @@ package com.mesalab.qgw.benchmark; +import cn.hutool.core.io.IORuntimeException; import cn.hutool.core.io.file.FileReader; +import cn.hutool.core.io.file.FileWriter; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.google.common.collect.Collections2; +import com.alibaba.nacos.client.config.impl.LocalConfigInfoProcessor; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.jayway.jsonpath.JsonPath; -import com.zdjizhi.utils.CommonUtil; +import com.mesalab.common.nacos.NacosConst; import com.zdjizhi.utils.StringUtil; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; -import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Map; -public abstract class Writer -{ +@Component +public abstract class Writer { + private static final Log log = LogFactory.get(); public static final boolean IS_TITLE = false; public static final String TITLE_PREFIX = "--Q"; @@ -25,11 +28,9 @@ public abstract class Writer public static final String LABEL_TIME_START = "@start"; public static final String LABEL_TIME_END = "@end"; - public Map<String, List<String>> getFilterOptions(String filterFile) { + public Map<String, List<String>> getFilterOptions(String jsonString) { Map<String, List<String>> environment = Maps.newLinkedHashMap(); try { - FileReader filterReader = new FileReader(filterFile); - String jsonString = filterReader.readString(); List<String> startTime = JsonPath.read(jsonString, "$.filters[?(@.name=='@start')].value"); List<String> endTime = JsonPath.read(jsonString, "$.filters[?(@.name=='@end')].value"); environment.put(LABEL_TIME_START, startTime); @@ -48,9 +49,86 @@ public abstract class Writer return environment; } - public abstract String buildPocSQL(); + protected String generatePocSQL(Map<String, List<String>> environment, FileReader templateReader, FileWriter writer) { + String pocSQL = ""; + try { + if (writer.getFile().exists()) { + return writer.getFile().getAbsolutePath(); + } + + List<String> rawLines = templateReader.readLines(); + List<String> resultLines = Lists.newArrayList(); + rawLines.stream().forEach(line -> { + + if (line.contains(TITLE_PREFIX)) { + if (IS_TITLE) { + resultLines.add(line); + } + } else { + + if (line.contains(LABEL_COMMON_FILTER)) { + for (String filter : environment.get(LABEL_COMMON_FILTER)) { + String replaceLine = line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) + .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0)) + .replaceAll(LABEL_COMMON_FILTER, filter); + + resultLines.add(replaceLine); + + } + + } else if (line.contains(LABEL_INDEX_FILTER)) { + for (String filter : environment.get(LABEL_INDEX_FILTER)) { + String replaceLine = line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) + .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0)) + .replaceAll(LABEL_INDEX_FILTER, filter); + + resultLines.add(replaceLine); + + } + + } else { + resultLines.add( + line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) + .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0))); + } + + + } + + }); + writer.appendLines(resultLines); + resultLines.clear(); + pocSQL = writer.getFile().getAbsolutePath(); + } catch (IORuntimeException e) { + log.error("IORuntimeException:{}", e); + } + return pocSQL; + } + + @Value("${nacos.config.server-addr}") + private String nacosServerAddr; + + protected String buildSnapshotFilePath(String dataId) { + String localSnapshotPath = LocalConfigInfoProcessor.LOCAL_SNAPSHOT_PATH.replaceAll("\\\\", "/"); + StringBuilder filePath = new StringBuilder(localSnapshotPath); + filePath.append("/"); + filePath.append(NacosConst.FIXED); + filePath.append(nacosServerAddr.replaceAll(":", "_")).append("-"); + filePath.append(NacosConst.NAMESPACE_GALAXY).append(NacosConst.NACOS); + filePath.append("/"); + filePath.append(NacosConst.SNAPSHOT_TENANT); + filePath.append("/"); + filePath.append(NacosConst.NAMESPACE_GALAXY); + filePath.append("/"); + filePath.append(NacosConst.DEFAULT_GROUP); + filePath.append("/"); + filePath.append(dataId); + log.info("dataId:{} snapshot path:{}", dataId, filePath.toString()); + return filePath.toString(); + } + protected abstract String buildPocSQL(); } diff --git a/src/main/java/com/mesalab/qgw/benchmark/clickhouse/ClickhouseWriter.java b/src/main/java/com/mesalab/qgw/benchmark/clickhouse/ClickhouseWriter.java index b15c90a8..dcebb5b0 100644 --- a/src/main/java/com/mesalab/qgw/benchmark/clickhouse/ClickhouseWriter.java +++ b/src/main/java/com/mesalab/qgw/benchmark/clickhouse/ClickhouseWriter.java @@ -4,87 +4,101 @@ import cn.hutool.core.io.file.FileReader; import cn.hutool.core.io.file.FileWriter; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.jayway.jsonpath.JsonPath; +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.config.listener.Listener; +import com.alibaba.nacos.api.exception.NacosException; +import com.mesalab.common.nacos.NacosConfig; +import com.mesalab.common.nacos.NacosConst; import com.mesalab.qgw.benchmark.Writer; import com.zdjizhi.utils.DateUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; - +@Component public class ClickhouseWriter extends Writer { - private static final Log log = LogFactory.get(); - public String buildPocSQL() { - String pocSql = ""; - try { - Map<String, List<String>> environment = getFilterOptions(new File("").getCanonicalPath() - + File.separator + "benchmark" + File.separator + "ck_filter.json"); - - FileReader templateReader = new FileReader(new File("").getCanonicalPath() - + File.separator + "benchmark" + File.separator + "ck_queries_template.sql"); - - FileWriter writer = new FileWriter(new File("").getCanonicalPath() - + File.separator + "benchmark" + File.separator + "ck_queries" + "_" - + DateUtils.getCurrentDate(DateUtils.YYYYMMDD) + ".sql"); - if (writer.getFile().exists()) { - return writer.getFile().getAbsolutePath(); - } + private static final Log log = LogFactory.get(); - List<String> rawLines = templateReader.readLines(); - List<String> resultLines = Lists.newArrayList(); - rawLines.stream().forEach(line -> { + private String ckFilterJson; - if (line.contains(TITLE_PREFIX)) { - if(IS_TITLE) { - resultLines.add(line); - } - } else { + @Autowired + private NacosConfig nacosConfig; - if (line.contains(LABEL_COMMON_FILTER)) { - for (String filter : environment.get(LABEL_COMMON_FILTER)) { - String replaceLine = line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) - .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0)) - .replaceAll(LABEL_COMMON_FILTER, filter); + @PostConstruct + public void init() { + initFilterJson(); + log.info("Initializing ClickHouse Filter Complete"); + initQueriesTemplate(); + log.info("Initializing ClickHouse Queries Template Complete"); + } - resultLines.add(replaceLine); + public void initFilterJson() { + try { + ConfigService configService = NacosFactory.createConfigService(nacosConfig.getProperties(NacosConst.NAMESPACE_GALAXY)); + ckFilterJson = configService.getConfig(NacosConst.CK_FILTER_DATA_ID, NacosConst.DEFAULT_GROUP, 3000); + configService.addListener(NacosConst.CK_FILTER_DATA_ID, NacosConst.DEFAULT_GROUP, new Listener() { + @Override + public Executor getExecutor() { + return null; + } - } + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo ClickHouse Filter {}", configInfo); + ckFilterJson = configInfo; + } + }); + } catch (NacosException e) { + log.info("NacosException:{}", e); + } + } - } else if (line.contains(LABEL_INDEX_FILTER)){ - for (String filter : environment.get(LABEL_INDEX_FILTER)) { - String replaceLine = line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) - .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0)) - .replaceAll(LABEL_INDEX_FILTER, filter); + private void initQueriesTemplate() { + try { + ConfigService configService = NacosFactory.createConfigService(nacosConfig.getProperties(NacosConst.NAMESPACE_GALAXY)); + configService.getConfig(NacosConst.CK_QUERIES_TEMPLATE_DATA_ID, NacosConst.DEFAULT_GROUP, 3000); + configService.addListener(NacosConst.CK_QUERIES_TEMPLATE_DATA_ID, NacosConst.DEFAULT_GROUP, new Listener() { + @Override + public Executor getExecutor() { + return null; + } - resultLines.add(replaceLine); + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo ClickHouse Queries Template {}", configInfo); + } + }); + } catch (NacosException e) { + log.info("NacosException:{}", e); + } + } - } + public String buildPocSQL() { + String pocSql = ""; + try { + Map<String, List<String>> environment = getFilterOptions(ckFilterJson); - } else { - resultLines.add( - line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) - .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0))); - } + FileReader templateReader = new FileReader(buildSnapshotFilePath(NacosConst.CK_QUERIES_TEMPLATE_DATA_ID)); + FileWriter writer = new FileWriter(new File("").getCanonicalPath() + + File.separator + "benchmark" + File.separator + "ck_queries" + "_" + + DateUtils.getCurrentDate(DateUtils.YYYYMMDD) + ".sql"); + pocSql = generatePocSQL(environment, templateReader, writer); - } - }); - writer.appendLines(resultLines); - resultLines.clear(); - pocSql = writer.getFile().getAbsolutePath(); } catch (IOException e) { log.error("Not found file {}", e); } - return pocSql; + return pocSql; } - - } diff --git a/src/main/java/com/mesalab/qgw/benchmark/druid/DruidWriter.java b/src/main/java/com/mesalab/qgw/benchmark/druid/DruidWriter.java index a7bebeb1..9f56ce85 100644 --- a/src/main/java/com/mesalab/qgw/benchmark/druid/DruidWriter.java +++ b/src/main/java/com/mesalab/qgw/benchmark/druid/DruidWriter.java @@ -4,83 +4,100 @@ import cn.hutool.core.io.file.FileReader; import cn.hutool.core.io.file.FileWriter; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.google.common.collect.Lists; +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.config.listener.Listener; +import com.alibaba.nacos.api.exception.NacosException; +import com.mesalab.common.nacos.NacosConfig; +import com.mesalab.common.nacos.NacosConst; import com.mesalab.qgw.benchmark.Writer; import com.zdjizhi.utils.DateUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; +@Component public class DruidWriter extends Writer { - private static final Log log = LogFactory.get(); - @Override - public String buildPocSQL() { - String pocSql = ""; - try { - Map<String, List<String>> environment = getFilterOptions(new File("").getCanonicalPath() - + File.separator + "benchmark" + File.separator + "druid_filter.json"); - - FileReader templateReader = new FileReader(new File("").getCanonicalPath() - + File.separator + "benchmark" + File.separator + "druid_queries_template.sql"); - - FileWriter druidWriter = new FileWriter(new File("").getCanonicalPath() - + File.separator + "benchmark" + File.separator + "druid_queries" + "_" - + DateUtils.getCurrentDate(DateUtils.YYYYMMDD) + ".sql"); - - if (druidWriter.getFile().exists()) { - return druidWriter.getFile().getAbsolutePath(); - } - - List<String> rawLines = templateReader.readLines(); - List<String> resultLines = Lists.newArrayList(); - rawLines.stream().forEach(line -> { + private static final Log log = LogFactory.get(); - if (line.contains(TITLE_PREFIX)) { - if(IS_TITLE) { - resultLines.add(line); - } - } else { - if (line.contains(LABEL_COMMON_FILTER)) { - for (String filter : environment.get(LABEL_COMMON_FILTER)) { - String replaceLine = line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) - .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0)) - .replaceAll(LABEL_COMMON_FILTER, filter); + private String druidFilterJson; - resultLines.add(replaceLine); + @Autowired + private NacosConfig nacosConfig; - } + @PostConstruct + public void init() { + initFilterJson(); + log.info("Initializing Druid Filter Complete"); + initQueriesTemplate(); + log.info("Initializing Druid Queries Template Complete"); + } - } else if (line.contains(LABEL_INDEX_FILTER)){ - for (String filter : environment.get(LABEL_INDEX_FILTER)) { - String replaceLine = line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) - .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0)) - .replaceAll(LABEL_INDEX_FILTER, filter); + public void initFilterJson() { + try { + ConfigService configService = NacosFactory.createConfigService(nacosConfig.getProperties(NacosConst.NAMESPACE_GALAXY)); + druidFilterJson = configService.getConfig(NacosConst.DRUID_FILTER_DATA_ID, NacosConst.DEFAULT_GROUP, 3000); + configService.addListener(NacosConst.DRUID_FILTER_DATA_ID, NacosConst.DEFAULT_GROUP, new Listener() { + @Override + public Executor getExecutor() { + return null; + } - resultLines.add(replaceLine); + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo Druid Filter {}", configInfo); + druidFilterJson = configInfo; + } + }); + } catch (NacosException e) { + log.info("NacosException:{}", e); + } + } - } + private void initQueriesTemplate() { + try { + ConfigService configService = NacosFactory.createConfigService(nacosConfig.getProperties(NacosConst.NAMESPACE_GALAXY)); + configService.getConfig(NacosConst.DRUID_QUERIES_TEMPLATE_DATA_ID, NacosConst.DEFAULT_GROUP, 3000); + configService.addListener(NacosConst.DRUID_QUERIES_TEMPLATE_DATA_ID, NacosConst.DEFAULT_GROUP, new Listener() { + @Override + public Executor getExecutor() { + return null; + } - } else { - resultLines.add( - line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) - .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0))); - } + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo Druid Queries Template {}", configInfo); + } + }); + } catch (NacosException e) { + log.info("NacosException:{}", e); + } + } + @Override + public String buildPocSQL() { + String pocSql = ""; + try { + Map<String, List<String>> environment = getFilterOptions(druidFilterJson); + FileReader templateReader = new FileReader(buildSnapshotFilePath(NacosConst.DRUID_QUERIES_TEMPLATE_DATA_ID)); - } + FileWriter druidWriter = new FileWriter(new File("").getCanonicalPath() + + File.separator + "benchmark" + File.separator + "druid_queries" + "_" + + DateUtils.getCurrentDate(DateUtils.YYYYMMDD) + ".sql"); - }); - druidWriter.appendLines(resultLines); - resultLines.clear(); - pocSql = druidWriter.getFile().getAbsolutePath(); + pocSql = generatePocSQL(environment, templateReader, druidWriter); } catch (IOException e) { log.error("Not found file {}", e); } - return pocSql; + return pocSql; } } diff --git a/src/main/java/com/mesalab/qgw/benchmark/engine/EngineWriter.java b/src/main/java/com/mesalab/qgw/benchmark/engine/EngineWriter.java index 66f792cb..dc80d401 100644 --- a/src/main/java/com/mesalab/qgw/benchmark/engine/EngineWriter.java +++ b/src/main/java/com/mesalab/qgw/benchmark/engine/EngineWriter.java @@ -4,83 +4,101 @@ import cn.hutool.core.io.file.FileReader; import cn.hutool.core.io.file.FileWriter; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.google.common.collect.Lists; +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.config.listener.Listener; +import com.alibaba.nacos.api.exception.NacosException; +import com.mesalab.common.nacos.NacosConfig; +import com.mesalab.common.nacos.NacosConst; import com.mesalab.qgw.benchmark.Writer; import com.zdjizhi.utils.DateUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; +@Component public class EngineWriter extends Writer { - private static final Log log = LogFactory.get(); - - @Override - public String buildPocSQL() { - String pocSql = ""; - try { - Map<String, List<String>> environment = getFilterOptions(new File("").getCanonicalPath() - + File.separator + "benchmark" + File.separator + "engine_filter.json"); - - FileReader templateReader = new FileReader(new File("").getCanonicalPath() - + File.separator + "benchmark" + File.separator + "engine_queries_template.sql"); - - FileWriter writer = new FileWriter(new File("").getCanonicalPath() - + File.separator + "benchmark" + File.separator + "engine_queries" + "_" - + DateUtils.getCurrentDate(DateUtils.YYYYMMDD) + ".sql"); - if (writer.getFile().exists()) { - return writer.getFile().getAbsolutePath(); - } + private static final Log log = LogFactory.get(); - List<String> rawLines = templateReader.readLines(); - List<String> resultLines = Lists.newArrayList(); - rawLines.stream().forEach(line -> { - if (line.contains(TITLE_PREFIX)) { - if(IS_TITLE) { - resultLines.add(line); - } - } else { + private String engineFilterJson; - if (line.contains(LABEL_COMMON_FILTER)) { - for (String filter : environment.get(LABEL_COMMON_FILTER)) { - String replaceLine = line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) - .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0)) - .replaceAll(LABEL_COMMON_FILTER, filter); + @Autowired + private NacosConfig nacosConfig; - resultLines.add(replaceLine); + @PostConstruct + public void init() { + initFilterJson(); + log.info("Initializing Engine Filter Complete"); + initQueriesTemplate(); + log.info("Initializing Engine Queries Template Complete"); + } - } + private void initFilterJson() { + try { + ConfigService configService = NacosFactory.createConfigService(nacosConfig.getProperties(NacosConst.NAMESPACE_GALAXY)); + engineFilterJson = configService.getConfig(NacosConst.ENGINE_FILTER_DATA_ID, NacosConst.DEFAULT_GROUP, 3000); + configService.addListener(NacosConst.ENGINE_FILTER_DATA_ID, NacosConst.DEFAULT_GROUP, new Listener() { + @Override + public Executor getExecutor() { + return null; + } - } else if (line.contains(LABEL_INDEX_FILTER)){ - for (String filter : environment.get(LABEL_INDEX_FILTER)) { - String replaceLine = line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) - .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0)) - .replaceAll(LABEL_INDEX_FILTER, filter); + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo Engine Filter {}", configInfo); + engineFilterJson = configInfo; + } + }); + } catch (NacosException e) { + log.info("NacosException:{}", e); + } + } - resultLines.add(replaceLine); + private void initQueriesTemplate() { + try { + ConfigService configService = NacosFactory.createConfigService(nacosConfig.getProperties(NacosConst.NAMESPACE_GALAXY)); + configService.getConfig(NacosConst.ENGINE_QUERIES_TEMPLATE_DATA_ID, NacosConst.DEFAULT_GROUP, 3000); + configService.addListener(NacosConst.ENGINE_QUERIES_TEMPLATE_DATA_ID, NacosConst.DEFAULT_GROUP, new Listener() { + @Override + public Executor getExecutor() { + return null; + } - } + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo Engine Queries Template {}", configInfo); + } + }); + } catch (NacosException e) { + log.info("NacosException:{}", e); + } + } - } else { - resultLines.add( - line.replaceAll(LABEL_TIME_START, environment.get(LABEL_TIME_START).get(0)) - .replaceAll(LABEL_TIME_END, environment.get(LABEL_TIME_END).get(0))); - } + @Override + public String buildPocSQL() { + String pocSql = ""; + try { + Map<String, List<String>> environment = getFilterOptions(engineFilterJson); + FileReader templateReader = new FileReader(buildSnapshotFilePath(NacosConst.ENGINE_QUERIES_TEMPLATE_DATA_ID)); + FileWriter writer = new FileWriter(new File("").getCanonicalPath() + + File.separator + "benchmark" + File.separator + "engine_queries" + "_" + + DateUtils.getCurrentDate(DateUtils.YYYYMMDD) + ".sql"); - } + pocSql = generatePocSQL(environment, templateReader, writer); - }); - writer.appendLines(resultLines); - resultLines.clear(); - pocSql = writer.getFile().getAbsolutePath(); } catch (IOException e) { log.error("Not found file {}", e); } - return pocSql; + return pocSql; } } diff --git a/src/main/java/com/mesalab/qgw/controller/MetadataController.java b/src/main/java/com/mesalab/qgw/controller/MetadataController.java index bc2db369..4d3592a4 100644 --- a/src/main/java/com/mesalab/qgw/controller/MetadataController.java +++ b/src/main/java/com/mesalab/qgw/controller/MetadataController.java @@ -5,17 +5,18 @@ import com.mesalab.common.base.BaseResultGenerator; import com.mesalab.qgw.service.MetadataService; import com.zdjizhi.utils.StringUtil; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.annotation.Resource; + @Slf4j @RestController @RequestMapping(value = "/metadata") public class MetadataController { - @Autowired + @Resource(name = "nacosMetadataService") private MetadataService metadataService; diff --git a/src/main/java/com/mesalab/qgw/controller/MonitorController.java b/src/main/java/com/mesalab/qgw/controller/MonitorController.java index 952679f0..a82fafd1 100644 --- a/src/main/java/com/mesalab/qgw/controller/MonitorController.java +++ b/src/main/java/com/mesalab/qgw/controller/MonitorController.java @@ -1,12 +1,10 @@ package com.mesalab.qgw.controller; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; +import com.mesalab.qgw.model.api.VersionInfo; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.io.*; - /** * @author wangwei * @description: @@ -15,35 +13,12 @@ import java.io.*; @RestController @RequestMapping(value = "/monitor") public class MonitorController { - private static final Log log = LogFactory.get(); - private static final String VERSION_FILE = "config" + File.separator + "version.json"; - @RequestMapping("/info") - public String config() { - String jsonStr = ""; - Reader reader = null; - try { - String pathRoot = new File("").getCanonicalPath(); - File jsonFile = new File(pathRoot + File.separator + VERSION_FILE); - reader = new InputStreamReader(new FileInputStream(jsonFile), "utf-8"); - int ch; - StringBuffer sb = new StringBuffer(); - while ((ch = reader.read()) != -1) { - sb.append((char) ch); - } - jsonStr = sb.toString(); - } catch (IOException e) { - log.error("Get file error: ", e); - } finally { + @Autowired + private VersionInfo version; - if (reader != null) { - try { - reader.close(); - } catch (IOException e) { - log.error("Close Reader error: ", e); - } - } - } - return jsonStr; + @RequestMapping("/info") + public VersionInfo config() { + return version; } } diff --git a/src/main/java/com/mesalab/qgw/controller/TestController.java b/src/main/java/com/mesalab/qgw/controller/TestController.java index 28f53d77..0050846c 100644 --- a/src/main/java/com/mesalab/qgw/controller/TestController.java +++ b/src/main/java/com/mesalab/qgw/controller/TestController.java @@ -2,7 +2,11 @@ package com.mesalab.qgw.controller; import com.mesalab.common.base.BaseResult; import com.mesalab.common.base.BaseResultGenerator; -import com.mesalab.common.configuration.ProjectProperties; +import com.mesalab.knowledge.common.config.ArangoConfig; +import com.mesalab.knowledge.common.utils.HttpConfig; +import com.mesalab.qgw.model.api.ProjectInfo; +import com.mesalab.common.enums.EnvironmentEnum; +import com.mesalab.qgw.model.api.*; import com.mesalab.qgw.service.DiagnosisService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -25,18 +29,26 @@ import java.util.Arrays; @RequestMapping(value = "test") public class TestController { - private final ProjectProperties projectProperties; - - private final Environment environment; - + @Autowired + private Environment environment; @Autowired private DiagnosisService diagnosisService; - @Autowired - public TestController(ProjectProperties projectProperties, Environment environment) { - this.projectProperties = projectProperties; - this.environment = environment; - } + private ProjectInfo projectProperties; + @Autowired + private JobAdminHttpSource jobAdminHttpSource; + @Autowired + private DruidIoHttpSource druidIoHttpSource; + @Autowired + private ClickHouseHttpSource clickHouseHttpSource; + @Autowired + private HBaseAPISource hBaseAPISource; + @Autowired + private EngineConfigSource engineConfigSource; + @Autowired + private ArangoConfig arangoConfig; + @Autowired + private HttpConfig httpConfig; /** * 测试 @@ -47,9 +59,7 @@ public class TestController { @GetMapping(value = "projectProperties") public BaseResult projectProperties() { log.info("Project Properties: {}", projectProperties); - //此处本意是将projectProperties返回至前端，但是projectProperties对象是由spring注入而来，其中包含过多的动态代理数据， - //使用lombok的@Data注解处理BaseResult时，数据过大，返回报错，因此只返回success - return BaseResultGenerator.success(); + return BaseResultGenerator.success("ok", projectProperties); } /** @@ -71,9 +81,7 @@ public class TestController { */ @GetMapping(value = "env") public BaseResult env() { - String[] env = projectProperties.getEnv(); - log.info("Project env: {}", Arrays.toString(env)); - return BaseResultGenerator.success(env); + return BaseResultGenerator.success(environment.getActiveProfiles()); } /** @@ -83,7 +91,7 @@ public class TestController { */ @GetMapping(value = "isProduct") public BaseResult isProduct() { - boolean isProduct = projectProperties.isProduct(); + boolean isProduct = environment.acceptsProfiles(EnvironmentEnum.PROD.getName()); String msg = "Current Environment is" + (isProduct ? "" : " not") + " product"; return BaseResultGenerator.success(msg); } @@ -123,4 +131,74 @@ public class TestController { return result; } + /** + * 检查job动态连接配置 + * + * @return + */ + @GetMapping(value = "jobAdminHttpSource") + public BaseResult jobAdminHttpSource() { + return BaseResultGenerator.success("ok", jobAdminHttpSource); + } + + /** + * 检查druid动态连接配置 + * + * @return + */ + @GetMapping(value = "druidIoHttpSource") + public BaseResult druidIoHttpSource() { + return BaseResultGenerator.success("ok", druidIoHttpSource); + } + + /** + * 检查clickHouse动态连接配置 + * + * @return + */ + @GetMapping(value = "clickHouseHttpSource") + public BaseResult clickHouseHttpSource() { + return BaseResultGenerator.success("ok", clickHouseHttpSource); + } + + /** + * 检查hBase动态连接配置 + * + * @return + */ + @GetMapping(value = "hBaseAPISource") + public BaseResult hBaseAPISource() { + return BaseResultGenerator.success("ok", hBaseAPISource); + } + + /** + * 检查engine动态连接配置 + * + * @return + */ + @GetMapping(value = "engineConfigSource") + public BaseResult engineConfigSource() { + return BaseResultGenerator.success("ok", engineConfigSource); + } + + /** + * 检查arango动态连接配置 + * + * @return + */ + @GetMapping(value = "arangoConfig") + public BaseResult arangoConfig() { + return BaseResultGenerator.success("ok", arangoConfig); + } + + /** + * 检查httpConfig动态连接配置 + * + * @return + */ + @GetMapping(value = "httpConfig") + public BaseResult httpConfig() { + return BaseResultGenerator.success("ok", httpConfig); + } + } diff --git a/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java b/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java index e2e6ea6c..93131164 100644 --- a/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java +++ b/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java @@ -6,8 +6,10 @@ import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.jayway.jsonpath.JsonPath; import com.mesalab.common.base.BaseResult; import com.mesalab.common.base.BaseResultGenerator; +import com.mesalab.common.base.DataTypeMapping; import com.mesalab.common.enums.*; import com.mesalab.common.exception.BusinessException; import com.mesalab.common.utils.*; @@ -37,10 +39,7 @@ import org.apache.http.client.utils.URLEncodedUtils; import org.springframework.util.CollectionUtils; import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -60,7 +59,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { private static final Pattern pCount1 = Pattern.compile("count\\(\\s*1\\s*\\)", Pattern.CASE_INSENSITIVE); private ClickHouseHttpSource clickHouseHttpSource = (ClickHouseHttpSource) SpringContextUtil.getBean("clickHouseHttpSource"); private HttpClientService httpClientService = (HttpClientService) SpringContextUtil.getBean("httpClientService"); - private MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("metadataService"); + private MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("nacosMetadataService"); private EngineConfigSource engineConfigSource = (EngineConfigSource) SpringContextUtil.getBean("engineConfigSource"); private HttpConfig httpConfig = (HttpConfig) SpringContextUtil.getBean("httpConfig"); private SystemService systemService = (SystemService) SpringContextUtil.getBean("systemService"); @@ -70,6 +69,27 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { private final static double PERCENT_UP_LINE = 0.98; private final static double PERCENT_DOWN_LINE = 0.01; + private final static Map<String, String> metaMap = Maps.newHashMap(); + + static { + //实际Int128、Int256、UInt64、UInt256的最大值已经超过long最大值 + metaMap.put("Int8", DataTypeMapping.INT); + metaMap.put("Int16", DataTypeMapping.INT); + metaMap.put("Int32", DataTypeMapping.INT); + metaMap.put("Int64", DataTypeMapping.LONG); + metaMap.put("Int128", DataTypeMapping.LONG); + metaMap.put("Int256", DataTypeMapping.LONG); + metaMap.put("UInt8", DataTypeMapping.INT); + metaMap.put("UInt16", DataTypeMapping.INT); + metaMap.put("UInt32", DataTypeMapping.LONG); + metaMap.put("UInt64", DataTypeMapping.LONG); + metaMap.put("UInt256", DataTypeMapping.LONG); + metaMap.put("Float32", DataTypeMapping.FLOAT); + metaMap.put("Float64", DataTypeMapping.DOUBLE); + metaMap.put("Date", DataTypeMapping.DATE); + metaMap.put("DateTime", DataTypeMapping.TIMESTAMP); + } + public ClickHouseDialect(ApiParam param) { this.param = param; } @@ -136,7 +156,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { List<NameValuePair> values = URLEncodedUtils.parse(queryParamBuilder.toString(), Charset.forName("UTF-8")); Map<String, String> map = httpClientService.httpGet(queryURL + URLEncodedUtils.format(values, "utf-8"), 0); - if (map.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { + if (Integer.parseInt(map.get("status")) == HttpStatus.SC_OK) { return BaseResultGenerator.generate(HttpStatus.SC_ACCEPTED, null, "ok", null, null, null, null); } else { return BaseResultGenerator.failure(Integer.parseInt(map.get("status")), map.get("result")); @@ -177,6 +197,16 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { return BaseResultGenerator.success(list); } + private List<Map<String, String>> unifyMeta(Object meta) { + List<Map<String, String>> metaList = Lists.newArrayList(); + if (StringUtil.isEmpty(meta)) { + return metaList; + } + metaList = JsonPath.read(meta, "$"); + metaList.forEach(x -> x.put("type", metaMap.get(x.get("type")) == null ? DataTypeMapping.STRING : metaMap.get(x.get("type")))); + return metaList; + } + class Converter { String sql = null; @@ -895,10 +925,10 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { Map<String, String> results = executeHttpGet(sql); - if (results.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { + if (Integer.parseInt(results.get("status")) == HttpStatus.SC_OK) { baseResult = BaseResultGenerator.generate( - Integer.valueOf(results.get("status")), + Integer.parseInt(results.get("status")), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), message.get(), JsonMapper.fromJsonString(results.get("result"), Map.class), @@ -906,7 +936,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { null, QueryFormatEnum.JSON.getValue()); } else { - baseResult = BaseResultGenerator.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), message.get(), + baseResult = BaseResultGenerator.generate(Integer.parseInt(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), message.get(), results.get("result"), null, null, param.getFormat()); } return build(); @@ -926,7 +956,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { statistics.put("elapsed", Math.round(Double.parseDouble(String.valueOf(statisticMap.get("elapsed"))) * 1000)); statistics.put("rows_read", statisticMap.get("rows_read")); baseResult.setData(resultMap.get("data")); - baseResult.setMeta(resultMap.get("meta")); + baseResult.setMeta(unifyMeta(resultMap.get("meta"))); baseResult.setStatistics(statistics); } else if (baseResult.getStatus().equals(HttpStatus.SC_NOT_FOUND)) { baseResult = BaseResultGenerator.failure(HttpStatus.SC_NOT_FOUND, @@ -1060,7 +1090,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { */ private FromItem generateDataSetSql(String tableName, String alias) throws JSQLParserException { String sql = String.format("SELECT 1 FROM (SELECT * FROM %s LIMIT %s ) as %s ", - tableName, 10000, alias); + tableName, 200, alias); Statement parse = CCJSqlParserUtil.parse(sql); Select select = (Select) parse; PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); diff --git a/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java b/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java index 34d66c50..18816003 100644 --- a/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java +++ b/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java @@ -5,9 +5,10 @@ import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.google.common.base.Stopwatch; import com.google.common.collect.Maps; -import com.mesalab.calcite.storage.DataTypeMapping; +import com.jayway.jsonpath.JsonPath; import com.mesalab.common.base.BaseResult; import com.mesalab.common.base.BaseResultGenerator; +import com.mesalab.common.base.DataTypeMapping; import com.mesalab.common.enums.*; import com.mesalab.common.exception.BusinessException; import com.mesalab.common.utils.*; @@ -26,6 +27,7 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Table; import net.sf.jsqlparser.statement.Statement; import net.sf.jsqlparser.statement.select.*; +import org.apache.commons.compress.utils.Lists; import org.apache.http.HttpStatus; import java.util.*; @@ -106,8 +108,8 @@ public class DruidDialect extends AbstractDataSourceDialect { url = url.substring(0, url.indexOf("/")); String deleteUrl = URLUtil.normalize(url).concat("/druid/v2/").concat(param.getQueryId()); Map<String, String> map = httpClientService.httpDelete(deleteUrl, 1000); - if (map.get("status").equals(String.valueOf(HttpStatus.SC_OK)) - || map.get("status").equals(String.valueOf(HttpStatus.SC_ACCEPTED))) { + if (Integer.parseInt(map.get("status")) == HttpStatus.SC_OK + || Integer.parseInt(map.get("status")) == HttpStatus.SC_ACCEPTED) { return BaseResultGenerator.generate(HttpStatus.SC_ACCEPTED, null, "ok", null, null, null, null); } else { return BaseResultGenerator.failure(Integer.parseInt(map.get("status")), map.get("result")); @@ -150,11 +152,11 @@ public class DruidDialect extends AbstractDataSourceDialect { Map<String, String> results = executeHttpPost(sql); - if (results.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { + if (Integer.parseInt(results.get("status")) == HttpStatus.SC_OK) { baseResult = BaseResultGenerator.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), message.get(), (List<Object>) JsonMapper.fromJsonString(results.get("result"), Object.class), null, null, QueryFormatEnum.JSON.getValue()); } else { - baseResult = BaseResultGenerator.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), results.get("message"), + baseResult = BaseResultGenerator.generate(Integer.parseInt(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), results.get("message"), results.get("result"), null, null, param.getFormat()); } @@ -182,7 +184,7 @@ public class DruidDialect extends AbstractDataSourceDialect { statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); statistics.put("rows_read", allResults.size()); baseResult.setData(allResults); - baseResult.setMeta(getMeta(allResults)); + baseResult.setMeta(unifyMeta(allResults)); baseResult.setStatistics(statistics); } else if (baseResult.getStatus().equals(HttpStatus.SC_NOT_FOUND)) { baseResult = BaseResultGenerator.failure(HttpStatus.SC_NOT_FOUND, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), @@ -252,25 +254,30 @@ public class DruidDialect extends AbstractDataSourceDialect { return plainSelect.getWhere(); } - private List getMeta(List<Object> results) { - List meta = new ArrayList<Map<String, String>>(); - if (results.size() > 0) { - Map<String, Object> map = (Map) results.get(0); - for (String key : map.keySet()) { - Object value = map.get(key); - Map<String, String> temp = new HashMap<>(); - temp.put("name", key); - if (value instanceof Integer) { - temp.put("type", DataTypeMapping.BIGINT); - } else if (value instanceof Double) { - temp.put("type", DataTypeMapping.DOUBLE); - } else { - temp.put("type", DataTypeMapping.VARCHAR); - } - meta.add(temp); + private List<Map<String, String>> unifyMeta(List<Object> list) { + List<Map<String, String>> metaList = Lists.newArrayList(); + if (StringUtil.isEmpty(list)) { + return metaList; + } + Map<String, Object> data = JsonPath.read(list.get(0), "$"); + for (String key : data.keySet()) { + Map<String, String> temp = Maps.newHashMap(); + temp.put("name", key); + Object value = data.get(key); + if (value instanceof Integer) { + temp.put("type", DataTypeMapping.INT); + } else if (value instanceof Long) { + temp.put("type", DataTypeMapping.LONG); + } else if (value instanceof Float) { + temp.put("type", DataTypeMapping.FLOAT); + } else if (value instanceof Double) { + temp.put("type", DataTypeMapping.DOUBLE); + } else { + temp.put("type", DataTypeMapping.STRING); } + metaList.add(temp); } - return meta; + return metaList; } class Converter { diff --git a/src/main/java/com/mesalab/qgw/model/api/ClickHouseHttpSource.java b/src/main/java/com/mesalab/qgw/model/api/ClickHouseHttpSource.java index 3cb53cff..708784cc 100644 --- a/src/main/java/com/mesalab/qgw/model/api/ClickHouseHttpSource.java +++ b/src/main/java/com/mesalab/qgw/model/api/ClickHouseHttpSource.java @@ -1,24 +1,35 @@ package com.mesalab.qgw.model.api; -import com.mesalab.common.configuration.ClickHouseProperties; +import com.alibaba.nacos.api.config.ConfigType; +import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; import lombok.Data; +import org.springframework.stereotype.Component; import java.io.Serializable; + @Data +@Component +@NacosConfigurationProperties(prefix = "clickhouse", dataId = "galaxy-qgw-service", type = ConfigType.YAML, autoRefreshed = true) public class ClickHouseHttpSource implements Serializable { private String url; private String dbName; - private ClickHouseProperties.DBAccount realTimeAccount ; + private DBAccount realTimeAccount; - private ClickHouseProperties.DBAccount longTermAccount; + private DBAccount longTermAccount; private int enableHttpCompression = 1; - private String systemDBName = "system"; + private String systemDBName = "system"; private long socketTimeout; + @Data + public static class DBAccount { + private String username; + private String userKey; + } + } diff --git a/src/main/java/com/mesalab/qgw/model/api/DruidIoHttpSource.java b/src/main/java/com/mesalab/qgw/model/api/DruidIoHttpSource.java index ca164543..5f11b0ea 100644 --- a/src/main/java/com/mesalab/qgw/model/api/DruidIoHttpSource.java +++ b/src/main/java/com/mesalab/qgw/model/api/DruidIoHttpSource.java @@ -1,10 +1,15 @@ package com.mesalab.qgw.model.api; +import com.alibaba.nacos.api.config.ConfigType; +import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; import lombok.Data; +import org.springframework.stereotype.Component; import java.io.Serializable; @Data +@Component +@NacosConfigurationProperties(prefix = "druid", dataId = "galaxy-qgw-service", type = ConfigType.YAML, autoRefreshed = true) public class DruidIoHttpSource implements Serializable { private String url; private String dbname; diff --git a/src/main/java/com/mesalab/qgw/model/api/EngineConfigSource.java b/src/main/java/com/mesalab/qgw/model/api/EngineConfigSource.java index 417373d0..9abdc3c9 100644 --- a/src/main/java/com/mesalab/qgw/model/api/EngineConfigSource.java +++ b/src/main/java/com/mesalab/qgw/model/api/EngineConfigSource.java @@ -1,10 +1,15 @@ package com.mesalab.qgw.model.api; +import com.alibaba.nacos.api.config.ConfigType; +import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; import lombok.Data; +import org.springframework.stereotype.Component; import java.io.Serializable; @Data +@Component +@NacosConfigurationProperties(prefix = "engine", dataId = "galaxy-qgw-service", type = ConfigType.YAML, autoRefreshed = true) public class EngineConfigSource implements Serializable { private int maxCacheNum; diff --git a/src/main/java/com/mesalab/qgw/model/api/HBaseAPISource.java b/src/main/java/com/mesalab/qgw/model/api/HBaseAPISource.java index 9cf45ab5..112fb205 100644 --- a/src/main/java/com/mesalab/qgw/model/api/HBaseAPISource.java +++ b/src/main/java/com/mesalab/qgw/model/api/HBaseAPISource.java @@ -1,19 +1,24 @@ package com.mesalab.qgw.model.api; +import com.alibaba.nacos.api.config.ConfigType; +import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; import lombok.Data; +import org.springframework.stereotype.Component; import java.io.Serializable; + @Data +@Component +@NacosConfigurationProperties(prefix = "hbase", dataId = "galaxy-qgw-service", type = ConfigType.YAML, autoRefreshed = true) public class HBaseAPISource implements Serializable { private String zookeeperQuorum; private String zookeeperPropertyClientPort; - private String zookeeperParent; + private String zookeeperZnodeParent; private String dbName; private String tableName; private String columnFamily; private String columnName; - private String clientRetriesNumber; private String clientIpcPoolSize; private String rpcTimeout; diff --git a/src/main/java/com/mesalab/qgw/model/api/JobAdminHttpSource.java b/src/main/java/com/mesalab/qgw/model/api/JobAdminHttpSource.java index 15301a37..637b15cb 100644 --- a/src/main/java/com/mesalab/qgw/model/api/JobAdminHttpSource.java +++ b/src/main/java/com/mesalab/qgw/model/api/JobAdminHttpSource.java @@ -1,10 +1,15 @@ package com.mesalab.qgw.model.api; +import com.alibaba.nacos.api.config.ConfigType; +import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; import lombok.Data; +import org.springframework.stereotype.Component; import java.io.Serializable; @Data +@Component +@NacosConfigurationProperties(prefix = "xxl-job-admin", dataId = "galaxy-qgw-service", type = ConfigType.YAML, autoRefreshed = true) public class JobAdminHttpSource implements Serializable { private String url; diff --git a/src/main/java/com/mesalab/qgw/model/api/ProjectInfo.java b/src/main/java/com/mesalab/qgw/model/api/ProjectInfo.java new file mode 100644 index 00000000..8bf9b188 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/api/ProjectInfo.java @@ -0,0 +1,72 @@ +package com.mesalab.qgw.model.api; + +import com.alibaba.nacos.api.config.ConfigType; +import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; +import com.mesalab.common.enums.EnvironmentEnum; +import com.mesalab.common.enums.EnvironmentGroupEnum; +import lombok.Data; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * 项目配置 + * + * @author dazzlzy + * @date 2018/5/26 + */ +@Data +@Component +@NacosConfigurationProperties(prefix = "project", dataId = "galaxy-qgw-service", type = ConfigType.YAML, autoRefreshed = true) +public class ProjectInfo { + + /** + * 工程名 + */ + private String name; + + /** + * 版本 + */ + private String version; + + /** + * 工程描述 + */ + private String description; + + /** + * 项目组织标识 + */ + private String groupId; + + /** + * 项目标识 + */ + private String artifactId; + + /** + * 项目根目录 + */ + private String basedir; + + /** + * 核心项目包 + */ + private String corePackage; + + /** + * 业务项目包 + */ + private String servicePackage; + + /** + * 项目作者 + */ + private Map author; + + +} diff --git a/src/main/java/com/mesalab/qgw/model/api/VersionInfo.java b/src/main/java/com/mesalab/qgw/model/api/VersionInfo.java new file mode 100644 index 00000000..76f26da9 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/api/VersionInfo.java @@ -0,0 +1,56 @@ +package com.mesalab.qgw.model.api; + +import com.alibaba.nacos.api.config.ConfigType; +import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; +import lombok.Data; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * @Date: 2021-03-12 11:27 + * @Author : liuyongqiang + * @ClassName : VersionInfo + * @Description : VersionInfo + */ +@Data +@Component +@NacosConfigurationProperties(dataId = "galaxy-qgw-version", type = ConfigType.JSON, autoRefreshed = true) +public class VersionInfo { + + private String product; + private String version; + private String registered; + private String updated; + private ComponentsBean components; + + @Data + public static class ComponentsBean { + private List<OssBean> oss; + private List<AppsBean> apps; + private List<TasksBean> tasks; + + @Data + public static class OssBean { + private String name; + private String version; + private String licenseType; + private String description; + } + + @Data + public static class AppsBean { + + private String name; + private String version; + private String description; + } + + @Data + public static class TasksBean { + private String name; + private String version; + private String description; + } + } +} diff --git a/src/main/java/com/mesalab/qgw/model/api/udf/TIME_FLOOR_WITH_FILL.java b/src/main/java/com/mesalab/qgw/model/api/udf/TIME_FLOOR_WITH_FILL.java index d8790f35..76b1e018 100644 --- a/src/main/java/com/mesalab/qgw/model/api/udf/TIME_FLOOR_WITH_FILL.java +++ b/src/main/java/com/mesalab/qgw/model/api/udf/TIME_FLOOR_WITH_FILL.java @@ -40,7 +40,7 @@ import java.util.*; public class TIME_FLOOR_WITH_FILL implements UDF{ private static final Log log = LogFactory.get(); @Autowired - private MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("metadataService"); + private MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("nacosMetadataService"); private EngineConfigSource engineConfigSource = (EngineConfigSource) SpringContextUtil.getBean("engineConfigSource"); private String name; private String timestamp; diff --git a/src/main/java/com/mesalab/qgw/model/metadata/MetadataBean.java b/src/main/java/com/mesalab/qgw/model/metadata/MetadataBean.java new file mode 100644 index 00000000..4800c0f6 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/metadata/MetadataBean.java @@ -0,0 +1,21 @@ +package com.mesalab.qgw.model.metadata; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * @Date: 2021-03-11 17:49 + * @Author : liuyongqiang + * @ClassName : MetadataBean + * @Description : MetadataBean + */ +@Data +public class MetadataBean implements Serializable { + + private String namespace; + private String group; + private List<String> tables; + +} diff --git a/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java index dfd6cca3..ddb61f89 100644 --- a/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java @@ -1,31 +1,20 @@ package com.mesalab.qgw.monitor; -import com.mesalab.common.configuration.HBaseProperties; -import com.mesalab.common.exception.BusinessException; import com.mesalab.knowledge.common.config.ArangoConfig; import com.mesalab.knowledge.common.utils.Constant; import com.mesalab.knowledge.common.utils.JwtCache; -import com.mesalab.knowledge.strategy.QueryProvider; import com.mesalab.qgw.service.impl.HttpClientService; import com.zdjizhi.utils.JsonMapper; import com.zdjizhi.utils.StringUtil; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.http.HttpStatus; import org.apache.http.message.BasicHeader; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.health.AbstractHealthIndicator; import org.springframework.boot.actuate.health.Health; import org.springframework.http.HttpHeaders; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; import java.util.HashMap; import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; @Component @@ -57,7 +46,7 @@ public class ArangoHealthIndicator extends AbstractHealthIndicator { } } String result = httpClientService.httpPost(arangoConfig.getServer() + "/_db/" + arangoConfig.getDatabase() + "/_api/version", - StringUtil.EMPTY, new BasicHeader(HttpHeaders.AUTHORIZATION, String.valueOf(jwt))); + StringUtil.EMPTY, new BasicHeader(HttpHeaders.AUTHORIZATION, String.valueOf(jwt))); Map map = (Map) JsonMapper.fromJsonString(result, Map.class); if (StringUtil.isEmpty(map) || StringUtil.isEmpty(map.get("version"))) { builder.down() diff --git a/src/main/java/com/mesalab/qgw/monitor/HbaseHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/HbaseHealthIndicator.java index 2e699fad..882c5cfa 100644 --- a/src/main/java/com/mesalab/qgw/monitor/HbaseHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/HbaseHealthIndicator.java @@ -1,6 +1,6 @@ package com.mesalab.qgw.monitor; -import com.mesalab.common.configuration.HBaseProperties; +import com.mesalab.qgw.model.api.HBaseAPISource; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; @@ -8,15 +8,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.health.AbstractHealthIndicator; import org.springframework.boot.actuate.health.Health; import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; @Component public class HbaseHealthIndicator extends AbstractHealthIndicator { - public RestTemplate restTemplate = new RestTemplate(); @Autowired - private HBaseProperties hBaseProperties; + private HBaseAPISource hBaseProperties; @Autowired private Connection hbaseConnection; diff --git a/src/main/java/com/mesalab/qgw/service/MetadataService.java b/src/main/java/com/mesalab/qgw/service/MetadataService.java index 16181e99..c56795ba 100644 --- a/src/main/java/com/mesalab/qgw/service/MetadataService.java +++ b/src/main/java/com/mesalab/qgw/service/MetadataService.java @@ -10,14 +10,16 @@ public interface MetadataService { /** * 查询schema信息 + * * @param type * @param name * @return */ - BaseResult getSchemaInfo(String type, String name); + BaseResult getSchemaInfo(String type, String name); /** * 根据表名获取partitionKey + * * @param tableName * @return */ @@ -25,6 +27,7 @@ public interface MetadataService { /** * Schema Doc中: 通过key获取value + * * @param tableName * @param key * @return @@ -33,6 +36,7 @@ public interface MetadataService { /** * 通过数据源，获取所属数据库类型 + * * @param tableName * @return */ @@ -41,17 +45,32 @@ public interface MetadataService { /** * 通过数据源，获取所对应数据库名称 + * * @param tableName * @return */ String getDBNameByTableName(String tableName); - /** * 获取所有已注册schema + * * @return */ Map<String, Schema> getAllSchema(); + /** + * 获取 ClickHouse schema + * + * @return + */ + Map<String, Schema> getClickHouseSchema(); + + /** + * 获取 Druid schema + * + * @return + */ + Map<String, Schema> getDruidSchema(); + } diff --git a/src/main/java/com/mesalab/qgw/service/impl/ApiServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/ApiServiceImpl.java index 41ffb433..001851ab 100644 --- a/src/main/java/com/mesalab/qgw/service/impl/ApiServiceImpl.java +++ b/src/main/java/com/mesalab/qgw/service/impl/ApiServiceImpl.java @@ -20,8 +20,6 @@ import com.google.common.collect.Maps; import com.mesalab.common.base.BaseResult; import com.mesalab.common.base.BaseResultGenerator; import com.mesalab.common.base.SchemaBase; -import com.mesalab.common.configuration.ClickHouseProperties; -import com.mesalab.common.configuration.DruidIoProperties; import com.mesalab.common.enums.*; import com.mesalab.common.exception.BusinessException; import com.mesalab.common.utils.ReportCacheUtils; @@ -31,6 +29,8 @@ import com.mesalab.common.utils.SQLSyntaxParserUtil; import com.mesalab.qgw.dialect.Dialect; import com.mesalab.qgw.dialect.FederationDialect; import com.mesalab.qgw.model.api.ApiParam; +import com.mesalab.qgw.model.api.ClickHouseHttpSource; +import com.mesalab.qgw.model.api.DruidIoHttpSource; import com.mesalab.qgw.model.api.SQLQuerySource; import com.mesalab.qgw.model.api.udf.UDF; import com.mesalab.qgw.model.api.udf.UDFElements; @@ -52,6 +52,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; +import javax.annotation.Resource; import java.lang.reflect.Constructor; import java.util.*; @@ -59,10 +60,10 @@ import java.util.*; public class ApiServiceImpl implements ApiService { private static final Log log = LogFactory.get(); @Autowired - private ClickHouseProperties clickHouseProperties; - @Autowired - private DruidIoProperties druidIoProperties; + private ClickHouseHttpSource clickHouseHttpSource; @Autowired + private DruidIoHttpSource druidIoHttpSource; + @Resource(name = "nacosMetadataService") private MetadataService metadataServiceTemp; private static MetadataService metadataService; @@ -101,8 +102,8 @@ public class ApiServiceImpl implements ApiService { baseResult = executeShowStatement(param, showStatement); } else { throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), - "SQL Syntax Error: Only support statement operations for select, show and describe.", null); - } + "SQL Syntax Error: Only support statement operations for select, show and describe.", null); + } return baseResult; } @@ -121,7 +122,7 @@ public class ApiServiceImpl implements ApiService { return baseResult; } else { log.warn("Long-Term Results not exist, execute real-time query: resultID-{}, sql-{}", - param.getResultId(), StringUtil.strip(param.getQuery()) ); + param.getResultId(), StringUtil.strip(param.getQuery())); } @@ -168,7 +169,7 @@ public class ApiServiceImpl implements ApiService { Dialect dialect; try { Class dialectClazz = Class.forName("com.mesalab.qgw.dialect." + - CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, param.getDbType())+"Dialect"); + CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, param.getDbType()) + "Dialect"); Constructor constructor = dialectClazz.getConstructor(ApiParam.class); dialect = (Dialect) constructor.newInstance(param); } catch (Exception e) { @@ -178,13 +179,12 @@ public class ApiServiceImpl implements ApiService { } - /** * 对请求参数进行验证 - * @param param - * query:查询语句，统一小写 - * format：输出格式，默认为JSON - * option：查询选项，默认为real-time + * + * @param param query:查询语句，统一小写 + * format：输出格式，默认为JSON + * option：查询选项，默认为real-time */ private String validateAndGetTableName(ApiParam param) { @@ -221,12 +221,13 @@ public class ApiServiceImpl implements ApiService { /** * Visitor 模式解析SQL + * * @param sql */ @Deprecated private String getTableNameFromDruid(String sql) { String tableName = ""; - try { + try { List<SQLStatement> statementList = SQLUtils.parseStatements(sql, JdbcConstants.MYSQL); SchemaStatVisitor statVisitor = SQLUtils.createSchemaStatVisitor(JdbcConstants.MYSQL); @@ -235,19 +236,20 @@ public class ApiServiceImpl implements ApiService { Map<TableStat.Name, TableStat> tables = statVisitor.getTables(); if (tables.keySet().size() > 0) { - TableStat.Name name = Iterables.getFirst(tables.keySet(), null); + TableStat.Name name = Iterables.getFirst(tables.keySet(), null); tableName = name.getName(); } } catch (Exception e) { - log.error("sqlParser error: ", e); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：", e); + log.error("sqlParser error: ", e); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：", e); } - return tableName; + return tableName; } /** * 递归解析SQLQuerySource ，拆分为数据库执行SQL与引擎执行的SQL source + * * @param sqlQuerySource SQL解析对象 */ public static void convertQueryRecursive(ApiParam param, SQLQuerySource sqlQuerySource) { @@ -289,12 +291,9 @@ public class ApiServiceImpl implements ApiService { } - - - /** - * * 解析SQL + * * @param sql * @return */ @@ -309,7 +308,7 @@ public class ApiServiceImpl implements ApiService { Select selectStatement = (Select) statement; sqlQuerySource.setSqlBody(String.valueOf(selectStatement.getSelectBody())); TablesNamesFinder tablesNamesFinder = new TablesNamesFinder(); - List<String> tableList = tablesNamesFinder.getTableList(selectStatement); + List<String> tableList = tablesNamesFinder.getTableList(selectStatement); if (StringUtil.isNotEmpty(tableList)) { String tableName = tableList.get(0); @@ -318,15 +317,14 @@ public class ApiServiceImpl implements ApiService { sqlQuerySource.setTableName(tableName); sqlQuerySource.setPartitionKey(metadataService.getPartitionKey(tableName)); } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error: Can't find table name." ); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error: Can't find table name."); } - SelectBody body = selectStatement.getSelectBody(); - if (body instanceof PlainSelect) { //单条查询 + if (body instanceof PlainSelect) { //单条查询 PlainSelect select = (PlainSelect) body; @@ -345,14 +343,14 @@ public class ApiServiceImpl implements ApiService { List<SelectItem> selectItemsList = select.getSelectItems(); if (StringUtil.isNotEmpty(selectItemsList)) { - String selectItemsString = Joiner.on(",").join(selectItemsList); + String selectItemsString = Joiner.on(",").join(selectItemsList); sqlQuerySource.setSelectItems(selectItemsString); } - for (SelectItem item: selectItemsList) { - if (item instanceof SelectExpressionItem) { - SelectExpressionItem expressionItem = ((SelectExpressionItem)item); + for (SelectItem item : selectItemsList) { + if (item instanceof SelectExpressionItem) { + SelectExpressionItem expressionItem = ((SelectExpressionItem) item); if (StringUtil.isNotEmpty(expressionItem.getAlias())) { String aliasName = expressionItem.getAlias().getName(); if (aliasName.startsWith("\"") && aliasName.endsWith("\"")) { @@ -374,13 +372,12 @@ public class ApiServiceImpl implements ApiService { List<OrderByElement> orderByList = select.getOrderByElements(); if (StringUtil.isNotEmpty(orderByList)) { - String orderByString = Joiner.on(",").join(orderByList); + String orderByString = Joiner.on(",").join(orderByList); sqlQuerySource.setOrderBy(orderByString); sqlQuerySource.setListOrderElement(orderByList); } - FromItem subItem = select.getFromItem(); if (subItem instanceof SubSelect) { @@ -397,7 +394,7 @@ public class ApiServiceImpl implements ApiService { if (limit != null) { if (StringUtil.isNotEmpty(limit.getOffset())) { - sqlQuerySource.setLimit(StringUtil.setDefaultIfEmpty(limit.getOffset() , 0)+ "," + limit.getRowCount()); + sqlQuerySource.setLimit(StringUtil.setDefaultIfEmpty(limit.getOffset(), 0) + "," + limit.getRowCount()); } else { sqlQuerySource.setLimit(String.valueOf(limit.getRowCount())); } @@ -408,7 +405,7 @@ public class ApiServiceImpl implements ApiService { SetOperationList setOperationList = (SetOperationList) body; List<SelectBody> selects = setOperationList.getSelects(); //暂时只解析第一个结构，不接受不相同的where - if(StringUtil.isNotEmpty(selects)) { + if (StringUtil.isNotEmpty(selects)) { SQLQuerySource parseSql = parserSQLByAst(selects.get(0).toString()); sqlQuerySource.setUdfSet(parseSql.getUdfSet()); sqlQuerySource.setWhereExpression(parseSql.getWhereExpression()); @@ -417,21 +414,19 @@ public class ApiServiceImpl implements ApiService { } sqlQuerySource.setLimit(UNVALID_LIMIT_DESC); } else { //其它暂不支持 - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：Only support statement operation as select."); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：Only support statement operation as select."); } - } else { log.warn("Not support DML Parser"); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error: Only support statement operation as select." ); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error: Only support statement operation as select."); } - } catch (Exception e) { log.error("sqlParser error: ", e); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error:", e); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error:", e); } @@ -474,6 +469,7 @@ public class ApiServiceImpl implements ApiService { /** * 获取自定义函数类 + * * @param fun * @param expressions * @return @@ -493,9 +489,9 @@ public class ApiServiceImpl implements ApiService { } - /** - * 对SQL语句进行预先解析,为语法树结构 + * 对SQL语句进行预先解析,为语法树结构 + * * @param sql * @return */ @@ -507,7 +503,7 @@ public class ApiServiceImpl implements ApiService { List<SQLStatement> statementList = SQLUtils.parseStatements(sql, dialectDbType); if (statementList.size() > 0) { SQLStatement statement = statementList.get(0); - if (statement instanceof SQLSelectStatement) { + if (statement instanceof SQLSelectStatement) { SQLSelectStatement sqlSelectStatement = (SQLSelectStatement) statement; SQLSelectQuery sqlSelectQuery = sqlSelectStatement.getSelect().getQuery(); // 非union的查询语句 @@ -515,7 +511,7 @@ public class ApiServiceImpl implements ApiService { SQLSelectQueryBlock sqlSelectQueryBlock = (SQLSelectQueryBlock) sqlSelectQuery; // 获取字段列表 List<SQLSelectItem> selectItems = sqlSelectQueryBlock.getSelectList(); - sqlQuerySource.setSelectItems( Arrays.toString(selectItems.toArray())); + sqlQuerySource.setSelectItems(Arrays.toString(selectItems.toArray())); // 获取表 SQLTableSource table = sqlSelectQueryBlock.getFrom(); @@ -524,11 +520,11 @@ public class ApiServiceImpl implements ApiService { sqlQuerySource.setTableName(((SQLExprTableSource) table).getName().getSimpleName()); // join多表 } else if (table instanceof SQLJoinTableSource) { - SQLTableSource sqlSubTableSource = ((SQLJoinTableSource)table).getLeft(); + SQLTableSource sqlSubTableSource = ((SQLJoinTableSource) table).getLeft(); sqlQuerySource.setTableName(((SQLExprTableSource) sqlSubTableSource).getName().getSimpleName()); // 子查询作为表 } else if (table instanceof SQLSubqueryTableSource) { - SQLSelectQuery sqlSubSelectQuery = ((SQLSubqueryTableSource)table).getSelect().getQuery(); + SQLSelectQuery sqlSubSelectQuery = ((SQLSubqueryTableSource) table).getSelect().getQuery(); sqlQuerySource.setTableName(getMultiSqlTableName(sqlSubSelectQuery)); } // 获取where条件 @@ -552,7 +548,7 @@ public class ApiServiceImpl implements ApiService { if (limit != null) { if (StringUtil.isNotEmpty(limit.getOffset())) { - sqlQuerySource.setLimit(StringUtil.setDefaultIfEmpty(limit.getOffset() , 0)+ "," + limit.getRowCount()); + sqlQuerySource.setLimit(StringUtil.setDefaultIfEmpty(limit.getOffset(), 0) + "," + limit.getRowCount()); } else { sqlQuerySource.setLimit(String.valueOf(limit.getRowCount())); } @@ -565,15 +561,14 @@ public class ApiServiceImpl implements ApiService { sqlQuerySource.setLimit(UNVALID_LIMIT_DESC); } } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：Only support statement operation as select."); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：Only support statement operation as select."); } - } } catch (Exception e) { log.error("sqlParser error: ", e); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：", e); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：", e); } @@ -583,18 +578,18 @@ public class ApiServiceImpl implements ApiService { private String getMultiSqlTableName(SQLSelectQuery sqlSubSelectQuery) { - SQLTableSource subTable = ((SQLSelectQueryBlock) sqlSubSelectQuery).getFrom(); + SQLTableSource subTable = ((SQLSelectQueryBlock) sqlSubSelectQuery).getFrom(); if (subTable instanceof SQLExprTableSource) { return ((SQLExprTableSource) subTable).getName().getSimpleName(); } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：Multi-level nested subqueries are not supported yet."); + throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：Multi-level nested subqueries are not supported yet."); } } - /** * 执行describe语句 + * * @param param * @param statement * @return @@ -630,6 +625,7 @@ public class ApiServiceImpl implements ApiService { /** * 执行show语句 + * * @param param * @param showStatement * @return @@ -639,10 +635,10 @@ public class ApiServiceImpl implements ApiService { if (showStatement != null && !"tables".equalsIgnoreCase(showStatement.getName())) { throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "SQL Syntax Error：不支持Show tables 之外的操作", null); } - if (clickHouseProperties.getDbname().equalsIgnoreCase(param.getSchema())) { + if (clickHouseHttpSource.getDbName().equalsIgnoreCase(param.getSchema())) { param.setDbType(DBTypeEnum.CLICKHOUSE.getValue()); param.setDialectDBType(DBTypeEnum.CLICKHOUSE.getValue()); - } else if (druidIoProperties.getDbname().equalsIgnoreCase(param.getSchema())) { + } else if (druidIoHttpSource.getDbname().equalsIgnoreCase(param.getSchema())) { param.setDbType(DBTypeEnum.DRUID.getValue()); param.setDialectDBType(DBTypeEnum.DRUID.getValue()); param.setQuery("SELECT TABLE_NAME AS name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'TABLE'"); @@ -671,27 +667,28 @@ public class ApiServiceImpl implements ApiService { /** * 获取所有表信息 + * * @return */ private BaseResult getAllTables() { //查ck ApiParam temp = new ApiParam(); temp.setQuery("show tables"); - temp.setSchema(clickHouseProperties.getDbname()); + temp.setSchema(clickHouseHttpSource.getDbName()); BaseResult ck = new BaseResult(); try { ck = executeShowStatement(temp, null); - }catch (Exception e){ + } catch (Exception e) { log.info("tables: 查询clickhouse异常", e); } //查druid BaseResult druid = new BaseResult(); temp = new ApiParam(); - temp.setSchema(druidIoProperties.getDbname()); + temp.setSchema(druidIoHttpSource.getDbname()); try { druid = executeShowStatement(temp, null); - }catch (Exception e){ + } catch (Exception e) { log.info("tables: 查询druid 异常", e); } //合并结果 @@ -699,28 +696,28 @@ public class ApiServiceImpl implements ApiService { List symbols = new ArrayList<>(); String dbName = ""; date.put("type", "enum"); - if(ck.getStatus() != null && ck.getStatus() == HttpStatus.SC_OK){ - Map ckDate = (Map)ck.getData(); + if (ck.getStatus() != null && ck.getStatus() == HttpStatus.SC_OK) { + Map ckDate = (Map) ck.getData(); dbName += (String) ckDate.get("name"); - symbols.addAll((List)ckDate.get("symbols")); + symbols.addAll((List) ckDate.get("symbols")); } - if(druid.getStatus() != null && druid.getStatus() == HttpStatus.SC_OK){ - Map druidDate = (Map)druid.getData(); - if(!"".equals(dbName)){ + if (druid.getStatus() != null && druid.getStatus() == HttpStatus.SC_OK) { + Map druidDate = (Map) druid.getData(); + if (!"".equals(dbName)) { dbName += ","; } dbName += (String) druidDate.get("name"); - symbols.addAll((List)druidDate.get("symbols")); + symbols.addAll((List) druidDate.get("symbols")); } date.put("name", dbName); date.put("symbols", symbols); - if(ck.getStatus() == HttpStatus.SC_OK){ + if (ck.getStatus() == HttpStatus.SC_OK) { ck.setData(date); return ck; - }else if(druid.getStatus() == HttpStatus.SC_OK){ + } else if (druid.getStatus() == HttpStatus.SC_OK) { druid.setData(date); return druid; - }else { + } else { throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "查询所有tables异常", null); } } @@ -761,7 +758,7 @@ public class ApiServiceImpl implements ApiService { date.put("type", "enum"); date.put("name", database); Iterator iterator = tables.iterator(); - while (iterator.hasNext()){ + while (iterator.hasNext()) { list.add(String.valueOf(iterator.next())); date.put("symbols", list); } diff --git a/src/main/java/com/mesalab/qgw/service/impl/DiagnosisServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/DiagnosisServiceImpl.java index 90b0dd5c..03aa2c6c 100644 --- a/src/main/java/com/mesalab/qgw/service/impl/DiagnosisServiceImpl.java +++ b/src/main/java/com/mesalab/qgw/service/impl/DiagnosisServiceImpl.java @@ -19,11 +19,13 @@ import com.mesalab.common.enums.DBTypeEnum; import com.mesalab.common.enums.QueryFormatEnum; import com.mesalab.common.enums.ResultCodeEnum; import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.utils.SchemaCacheUtils; import com.mesalab.knowledge.common.utils.HttpConfig; import com.mesalab.qgw.benchmark.clickhouse.ClickhouseWriter; import com.mesalab.qgw.benchmark.druid.DruidWriter; import com.mesalab.qgw.benchmark.engine.EngineWriter; import com.mesalab.qgw.service.DiagnosisService; +import com.mesalab.qgw.service.MetadataService; import com.zdjizhi.utils.JsonMapper; import com.zdjizhi.utils.StringUtil; import org.apache.avro.Schema; @@ -32,10 +34,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import javax.annotation.Resource; import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -47,9 +51,9 @@ import java.util.stream.Stream; public class DiagnosisServiceImpl implements DiagnosisService { private static final Log log = LogFactory.get(); private static final String CK_SCHEMA_DIR = File.separator + "config" - + File.separator + "avro" + File.separator + "clickhouse"; + + File.separator + "avro" + File.separator + "clickhouse"; private static final String DRUID_SCHEMA_DIR = File.separator + "config" - + File.separator + "avro" + File.separator + "druid"; + + File.separator + "avro" + File.separator + "druid"; private static final String AVRO_SUFFIX = ".avsc"; private static final String METADATA_PREFIX = "Metadata"; private static final String SCHEMA_PREFIX = "Schema"; @@ -61,13 +65,21 @@ public class DiagnosisServiceImpl implements DiagnosisService { private HttpConfig httpConfig; @Value("${server.port}") private int serverPort; - private ClickhouseWriter clickhouseWriter = new ClickhouseWriter(); - private EngineWriter engineWriter = new EngineWriter(); - private DruidWriter druidWriter = new DruidWriter(); + @Autowired + private ClickhouseWriter clickhouseWriter; + @Autowired + private EngineWriter engineWriter; + @Autowired + private DruidWriter druidWriter; + + @Resource(name = "nacosMetadataService") + private MetadataService metadataService; + private static final String localHostAddress = NetUtil.getLocalhostStr(); /** * 验证Schema格式是否正确 + * * @return */ @Override @@ -87,15 +99,16 @@ public class DiagnosisServiceImpl implements DiagnosisService { result.setFormatType(QueryFormatEnum.JSON.getValue()); } catch (Exception e) { throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, - ResultCodeEnum.ENGINE_PARSE_ERROR.getCode(), "Avro Schema Validation Fail:" , e); + ResultCodeEnum.ENGINE_PARSE_ERROR.getCode(), "Avro Schema Validation Fail:", e); } - + SchemaCacheUtils.removeAll(); return result; } /** * 验证schema与数据库表结构是否一致 + * * @return */ @Override @@ -114,13 +127,12 @@ public class DiagnosisServiceImpl implements DiagnosisService { result.setFormatType(QueryFormatEnum.JSON.getValue()); } catch (Exception e) { throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, - ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Metadata Validation Fail:" , e); + ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Metadata Validation Fail:", e); } return result; } - @Override public BaseResult runPocSQL(boolean isSaved) { BaseResult result = null; @@ -153,7 +165,7 @@ public class DiagnosisServiceImpl implements DiagnosisService { result.setFormatType(QueryFormatEnum.JSON.getValue()); } catch (Exception e) { throw new BusinessException(HttpStatus.SC_BAD_REQUEST, - ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), "Execute Poc SQL Fail:" , e); + ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), "Execute Poc SQL Fail:", e); } return result; } @@ -166,23 +178,21 @@ public class DiagnosisServiceImpl implements DiagnosisService { Map<String, String> dialectResult = Maps.newHashMap(); if (dialect.equalsIgnoreCase(DBTypeEnum.CLICKHOUSE.getValue())) { - testFile = Optional.of(new File(clickhouseWriter.buildPocSQL())); + testFile = Optional.of(new File(clickhouseWriter.buildPocSQL())); } else if (dialect.equalsIgnoreCase(DBTypeEnum.ENGINE.getValue())) { - testFile = Optional.of(new File(engineWriter.buildPocSQL())); + testFile = Optional.of(new File(engineWriter.buildPocSQL())); } else if (dialect.equalsIgnoreCase(DBTypeEnum.DRUID.getValue())) { - testFile = Optional.of(new File(druidWriter.buildPocSQL())); + testFile = Optional.of(new File(druidWriter.buildPocSQL())); } else { throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Illegal value about dialect."); } - try { Stopwatch watch = Stopwatch.createStarted(); - if (testFile.isPresent()) { if (!testFile.get().isFile()) { log.warn("UnKnow File " + testFile.get()); @@ -196,7 +206,7 @@ public class DiagnosisServiceImpl implements DiagnosisService { result.setFormatType(QueryFormatEnum.JSON.getValue()); } catch (Exception e) { throw new BusinessException(HttpStatus.SC_BAD_REQUEST, - ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Get Poc SQL Fail:" , e); + ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Get Poc SQL Fail:", e); } return result; } @@ -204,7 +214,8 @@ public class DiagnosisServiceImpl implements DiagnosisService { /** * 执行POC SQL测试集，并返回执行结果 - * @param file sql 查询语句列表 + * + * @param file sql 查询语句列表 * @param isSaved 是否保存查询结果 * @return */ @@ -217,50 +228,48 @@ public class DiagnosisServiceImpl implements DiagnosisService { if (file.isPresent() && file.get().isFile()) { if (isSaved) { - writer = new FileWriter( file.get().getAbsolutePath() + "." + IdUtil.simpleUUID() + ".dat" ); + writer = new FileWriter(file.get().getAbsolutePath() + "." + IdUtil.simpleUUID() + ".dat"); } for (String line : rawLines) { - index ++; + index++; if (Strings.isNullOrEmpty(line)) { continue; } Map<String, String> queryResult = executeQuery(line, Optional.of(QueryFormatEnum.CSV)); int statusCode = Integer.valueOf(queryResult.get("status").trim()); if (statusCode != HttpStatus.SC_OK) { - throw new BusinessException(statusCode, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), "Query" + index + " : " + JsonPath.read(queryResult.get("result"), "$.message" ), null); + throw new BusinessException(statusCode, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), "Query" + index + " : " + JsonPath.read(queryResult.get("result"), "$.message"), null); } if (writer != null) { - writer.append("--Query" + index + "\t" + "Elapsed(ms):" + JsonPath.read(queryResult.get("result"), "$.statistics.elapsed" ) + "\n"); - writer.append(JsonPath.read(queryResult.get("result"), "$.meta" ) + "\n"); - writer.append(JsonPath.read(queryResult.get("result"), "$.data" ) + "\n"); + writer.append("--Query" + index + "\t" + "Elapsed(ms):" + JsonPath.read(queryResult.get("result"), "$.statistics.elapsed") + "\n"); + writer.append(JsonPath.read(queryResult.get("result"), "$.meta") + "\n"); + writer.append(JsonPath.read(queryResult.get("result"), "$.data") + "\n"); } - infoMap.put("Query" + index, JsonPath.read(queryResult.get("result"), "$.statistics.elapsed" ) + ""); + infoMap.put("Query" + index, JsonPath.read(queryResult.get("result"), "$.statistics.elapsed") + ""); } resultMap.put(file.get().getName(), infoMap); } else { - log.error("File is not exist or is directory." ); + log.error("File is not exist or is directory."); } - - return resultMap; } - private Map<String, String> getPocSqlByDatasource(Optional<File> file, Integer queryNo) { + private Map<String, String> getPocSqlByDatasource(Optional<File> file, Integer queryNo) { Map<String, String> infoMap = Maps.newLinkedHashMap(); List<String> rawLines = new FileReader(file.get()).readLines(); if (StringUtil.isEmpty(queryNo)) { int index = 0; for (String line : rawLines) { - index ++ ; + index++; infoMap.put("Query" + index, line); } } else { - infoMap.put("Query" + queryNo, rawLines.get(queryNo-1)); + infoMap.put("Query" + queryNo, rawLines.get(queryNo - 1)); } return infoMap; } @@ -268,29 +277,37 @@ public class DiagnosisServiceImpl implements DiagnosisService { /** * 对指定目录下的schema 与数据库的字段进行对比检查 + * * @param path schema 列表 * @return */ private Map<String, Map<String, String>> checkMetadataByDatasource(String path) { - Map<String, Schema> schemas = getSchemasByDatasource(path); + Map<String, Schema> schemas = new HashMap<>(); + if (path.equals(CK_SCHEMA_DIR)) { + schemas = metadataService.getClickHouseSchema(); + } + if (path.equals(DRUID_SCHEMA_DIR)) { + schemas = metadataService.getDruidSchema(); + } + Map<String, Map<String, String>> resultMap = Maps.newLinkedHashMap(); Map<String, String> infoMap = Maps.newHashMap(); for (String tableName : schemas.keySet()) { Map<String, String> result = executeQuery("describe " + tableName, Optional.empty()); - List<Map<String,Object>> dbFields = JsonPath.read(result.get("result"),"$.data.fields.*"); + List<Map<String, Object>> dbFields = JsonPath.read(result.get("result"), "$.data.fields.*"); Map<String, Object> dbFieldMap = Maps.newHashMap(); - dbFields.stream().forEach( objectMap -> { + dbFields.stream().forEach(objectMap -> { dbFieldMap.put(objectMap.get("name").toString(), objectMap.get("type")); }); - List<Map<String,Object>> schemaFields = JsonPath.read(schemas.get(tableName).toString(), "$.fields.*"); + List<Map<String, Object>> schemaFields = JsonPath.read(schemas.get(tableName).toString(), "$.fields.*"); Map<String, Object> schemaFieldMap = Maps.newHashMap(); - schemaFields.stream().forEach( objectMap -> { + schemaFields.stream().forEach(objectMap -> { schemaFieldMap.put(objectMap.get("name").toString(), objectMap.get("type")); }); - MapDifference<String, Object> difference = Maps.difference(schemaFieldMap, dbFieldMap); + MapDifference<String, Object> difference = Maps.difference(schemaFieldMap, dbFieldMap); Map<String, Object> entriesOnlyOnLeft = difference.entriesOnlyOnLeft(); @@ -298,7 +315,7 @@ public class DiagnosisServiceImpl implements DiagnosisService { log.error("键只存在于左边Map的映射项:" + entriesOnlyOnLeft); throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.ENGINE_PARSE_ERROR.getCode(), - " Schema Fields greater than Table Fields,Schema " + tableName + " exist fields " + entriesOnlyOnLeft , null); + " Schema Fields greater than Table Fields,Schema " + tableName + " exist fields " + entriesOnlyOnLeft, null); } Map<String, Object> entriesOnlyOnRight = difference.entriesOnlyOnRight(); @@ -312,19 +329,19 @@ public class DiagnosisServiceImpl implements DiagnosisService { resultMap.put(METADATA_PREFIX + path, infoMap); - return resultMap; + return resultMap; } private Map<String, String> executeQuery(String sql, Optional<QueryFormatEnum> formatEnum) { Map<String, String> resultMap = Maps.newHashMap(); - try{ + try { if (serverPort == 0) { serverPort = DEFAULT_SERVER_PORT; } - QueryFormatEnum format = formatEnum.orElse(QueryFormatEnum.JSON); + QueryFormatEnum format = formatEnum.orElse(QueryFormatEnum.JSON); sql = URLEncoder.encode(sql, "utf-8").replaceAll("\\+", "%20"); String queryURL = URLUtil.normalize(localHostAddress + ":" + serverPort + "/?query="); int socketTimeOut = httpConfig.getServerResponseTimeOut(); @@ -337,19 +354,25 @@ public class DiagnosisServiceImpl implements DiagnosisService { } - /** * 对指定的数据源下的schema进行静态检查 + * * @param path * @return */ private Map<String, Map<String, String>> parseSchemaByDatasource(String path) { - Map<String, Schema> schemas = getSchemasByDatasource(path); + Map<String, Schema> schemas = new HashMap<>(); + if (path.equals(CK_SCHEMA_DIR)) { + schemas = metadataService.getClickHouseSchema(); + } + if (path.equals(DRUID_SCHEMA_DIR)) { + schemas = metadataService.getDruidSchema(); + } Map<String, Map<String, String>> resultMap = Maps.newLinkedHashMap(); Map<String, String> infoMap = Maps.newHashMap(); for (String tableName : schemas.keySet()) { Map schemaMap = (Map) JsonMapper.fromJsonString(schemas.get(tableName).toString(), Map.class); - List<Map<String,Object>> fields = (List<Map<String,Object>>) schemaMap.get("fields"); + List<Map<String, Object>> fields = (List<Map<String, Object>>) schemaMap.get("fields"); infoMap.put(tableName, fields.size() + " Fields"); } resultMap.put(SCHEMA_PREFIX + path, infoMap); @@ -360,6 +383,7 @@ public class DiagnosisServiceImpl implements DiagnosisService { /** * 获取当前路径下的schema对象列表 + * * @param path avro 路径 * @return */ @@ -370,21 +394,21 @@ public class DiagnosisServiceImpl implements DiagnosisService { if (FileUtil.isNotEmpty(schemaDir) && FileUtil.isDirectory(schemaDir)) { if (StringUtil.isEmpty(schemaDir.listFiles())) { throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, - ResultCodeEnum.ENGINE_PARSE_ERROR.getCode(), "The Directory is null: " + path) ; + ResultCodeEnum.ENGINE_PARSE_ERROR.getCode(), "The Directory is null: " + path); } for (File file : schemaDir.listFiles()) { - try { - if (StringUtil.lowerCase(file.getName()).endsWith(AVRO_SUFFIX)) { - Schema schema = new Schema.Parser().parse(file); - schemas.put(schema.getName(), schema); - } else { - log.warn("UnKnow format file {} ", file.getName()); - } - } catch (Exception e) { - log.error("Schema Parsing Error {}, Message is {}", file.getName(), e); - throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, - ResultCodeEnum.ENGINE_PARSE_ERROR.getCode(), file.getName() + " Parse Error: " + e.getMessage() , e); - } + try { + if (StringUtil.lowerCase(file.getName()).endsWith(AVRO_SUFFIX)) { + Schema schema = new Schema.Parser().parse(file); + schemas.put(schema.getName(), schema); + } else { + log.warn("UnKnow format file {} ", file.getName()); + } + } catch (Exception e) { + log.error("Schema Parsing Error {}, Message is {}", file.getName(), e); + throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, + ResultCodeEnum.ENGINE_PARSE_ERROR.getCode(), file.getName() + " Parse Error: " + e.getMessage(), e); + } } } } catch (IOException e) { @@ -394,7 +418,4 @@ public class DiagnosisServiceImpl implements DiagnosisService { } - - - } diff --git a/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java b/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java index 862fa15f..95aee831 100644 --- a/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java +++ b/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java @@ -144,8 +144,7 @@ public class HttpClientService { } /** - * Desc: 发起http delete请求 - * + * Desc: 发起http delete请求，返回status code与response body * @param url * @param socketTimeout * @return {@link Map< String, String>} @@ -195,8 +194,10 @@ public class HttpClientService { } /** - * @param url:请求地址 + * 返回status code与response body + * @param url:请求地址 * @param socketTimeout: 响应超时时间 + * **/ public Map<String, String> httpGet(String url, int socketTimeout) { Map<String, String> resultMap = Maps.newHashMap(); @@ -241,6 +242,7 @@ public class HttpClientService { } /** + * 返回status code与response body * @param url:请求地址 * @param headers: Headers * @param socketTimeOut: 响应超时时间 @@ -292,6 +294,7 @@ public class HttpClientService { } /** + * 返回status code与response body * @param url:请求地址 * @param jsonString:请求参数 * @param socketTimeOut:响应超时时间 @@ -340,6 +343,7 @@ public class HttpClientService { } /** + * 返回status code与response body * @param url:请求地址 * @param headers: Headers * @param socketTimeOut: 响应超时时间 diff --git a/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java index 9e4645cd..a6162e1f 100644 --- a/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java +++ b/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java @@ -4,13 +4,13 @@ import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.mesalab.common.base.BaseResult; import com.mesalab.common.base.BaseResultGenerator; -import com.mesalab.common.configuration.ClickHouseProperties; -import com.mesalab.common.configuration.DruidIoProperties; +import com.mesalab.common.enums.DBTypeEnum; import com.mesalab.common.enums.ResultCodeEnum; import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.enums.DBTypeEnum; import com.mesalab.common.exception.BusinessException; import com.mesalab.common.utils.SchemaCacheUtils; +import com.mesalab.qgw.model.api.ClickHouseHttpSource; +import com.mesalab.qgw.model.api.DruidIoHttpSource; import com.mesalab.qgw.service.MetadataService; import com.zdjizhi.utils.JsonMapper; import com.zdjizhi.utils.StringUtil; @@ -30,9 +30,9 @@ public class MetadataServiceImpl implements MetadataService { private static final String SCHEMA_PATH = "config" + File.separator + "avro"; @Autowired - ClickHouseProperties clickHouseProperties; + ClickHouseHttpSource clickHouseHttpSource; @Autowired - DruidIoProperties druidIoProperties; + DruidIoHttpSource druidIoHttpSource; @Override public BaseResult getSchemaInfo(String type, String name) { @@ -185,11 +185,11 @@ public class MetadataServiceImpl implements MetadataService { if (StringUtil.isEmpty(schema)) { return StringUtil.EMPTY; } - if (clickHouseProperties.getDbname().equalsIgnoreCase(schema.getNamespace()) + if (clickHouseHttpSource.getDbName().equalsIgnoreCase(schema.getNamespace()) || "system".equalsIgnoreCase(schema.getNamespace())) { return DBTypeEnum.CLICKHOUSE.getValue(); } - if (druidIoProperties.getDbname().equalsIgnoreCase(schema.getNamespace())) { + if (druidIoHttpSource.getDbname().equalsIgnoreCase(schema.getNamespace())) { return DBTypeEnum.DRUID.getValue(); } return StringUtil.EMPTY; @@ -198,7 +198,7 @@ public class MetadataServiceImpl implements MetadataService { @Override public String getDBNameByTableName(String tableName) { Schema schema = getSchemaByName(tableName); - return StringUtil.isNotEmpty(schema) ? schema.getNamespace() : clickHouseProperties.getDbname() ; + return StringUtil.isNotEmpty(schema) ? schema.getNamespace() : clickHouseHttpSource.getDbName(); } @@ -214,4 +214,14 @@ public class MetadataServiceImpl implements MetadataService { } } + + @Override + public Map<String, Schema> getClickHouseSchema() { + return null; + } + + @Override + public Map<String, Schema> getDruidSchema() { + return null; + } } diff --git a/src/main/java/com/mesalab/qgw/service/impl/NacosMetadataServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/NacosMetadataServiceImpl.java new file mode 100644 index 00000000..d663aec6 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/NacosMetadataServiceImpl.java @@ -0,0 +1,283 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.config.listener.Listener; +import com.alibaba.nacos.api.exception.NacosException; +import com.mesalab.common.base.BaseResult; +import com.mesalab.common.base.BaseResultGenerator; +import com.mesalab.common.enums.DBTypeEnum; +import com.mesalab.common.enums.MetadataTypeEnum; +import com.mesalab.common.enums.ResultCodeEnum; +import com.mesalab.common.nacos.NacosConfig; +import com.mesalab.common.nacos.NacosConst; +import com.mesalab.qgw.model.api.ClickHouseHttpSource; +import com.mesalab.qgw.model.api.DruidIoHttpSource; +import com.mesalab.qgw.model.metadata.MetadataBean; +import com.mesalab.qgw.service.MetadataService; +import com.zdjizhi.utils.JsonMapper; +import com.zdjizhi.utils.StringUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.Schema; +import org.apache.http.HttpStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.util.*; +import java.util.concurrent.Executor; + +/** + * @Date: 2021-03-11 15:48 + * @Author : liuyongqiang + * @ClassName : NacosMetadataServiceImpl + * @Description : 获取Nacos注册中心元数据实现类 + */ +@Slf4j +@Service("nacosMetadataService") +public class NacosMetadataServiceImpl implements MetadataService { + + @Autowired + private NacosConfig nacosConfig; + @Autowired + private ClickHouseHttpSource clickHouseHttpSource; + @Autowired + private DruidIoHttpSource druidIoHttpSource; + + public List<MetadataBean> metadataBeans; + + public Map<String, Schema> allSchemaMap = new HashMap<>(); + + public Map<String, Schema> clickHouseSchemaMap = new HashMap<>(); + + public Map<String, Schema> druidSchemaMap = new HashMap<>(); + + + @Override + public BaseResult getSchemaInfo(String type, String name) { + if (MetadataTypeEnum.TABLES.getValue().equals(type)) { + return loadTables(name); + } else if (MetadataTypeEnum.FIELDS.getValue().equals(type)) { + return loadFields(name); + } else { + return BaseResultGenerator.failure(HttpStatus.SC_NOT_FOUND, + ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "Not found, check url please"); + } + } + + @Override + public String getPartitionKey(String tableName) { + String partitionKey = StringUtil.EMPTY; + Schema schema = getSchemaByName(tableName); + Map doc = (Map) JsonMapper.fromJsonString(schema.getDoc(), Map.class); + if (StringUtil.isNotEmpty(doc)) { + Object partition_key = doc.get("partition_key"); + partitionKey = StringUtil.isEmpty(partition_key) ? StringUtil.EMPTY : partition_key.toString(); + } + return partitionKey; + } + + @Override + public String getValueByKeyInSchemaDoc(String tableName, String key) { + Schema schema = getSchemaByName(tableName); + Map doc = (Map) JsonMapper.fromJsonString(schema.getDoc(), Map.class); + if (StringUtil.isNotEmpty(doc)) { + Object value = doc.get(key); + if (value instanceof Map) { + return JsonMapper.toJsonString(value); + } else { + return StringUtil.isEmpty(value) ? StringUtil.EMPTY : value.toString(); + } + } + return StringUtil.EMPTY; + } + + @Override + public String getDBTypeByTableName(String tableName) { + Schema schema = getSchemaByName(tableName); + if (StringUtil.isEmpty(schema)) { + return StringUtil.EMPTY; + } + if (clickHouseHttpSource.getDbName().equalsIgnoreCase(schema.getNamespace()) + || "system".equalsIgnoreCase(schema.getNamespace())) { + return DBTypeEnum.CLICKHOUSE.getValue(); + } + if (druidIoHttpSource.getDbname().equalsIgnoreCase(schema.getNamespace())) { + return DBTypeEnum.DRUID.getValue(); + } + return StringUtil.EMPTY; + } + + @Override + public String getDBNameByTableName(String tableName) { + Schema schema = getSchemaByName(tableName); + return StringUtil.isNotEmpty(schema) ? schema.getNamespace() : clickHouseHttpSource.getDbName(); + } + + @Override + public Map<String, Schema> getAllSchema() { + return allSchemaMap; + } + + @Override + public Map<String, Schema> getClickHouseSchema() { + return clickHouseSchemaMap; + } + + @Override + public Map<String, Schema> getDruidSchema() { + return druidSchemaMap; + } + + @PostConstruct + public void init() { + initTables(); + log.info("Initializing Schema Tables Complete"); + initFields(); + log.info("Initializing Schema Fields Complete"); + } + + private void initTables() { + try { + ConfigService configService = NacosFactory.createConfigService(nacosConfig.getProperties(NacosConst.NAMESPACE_AVRO)); + String content = configService.getConfig(NacosConst.META_DATA_ID, NacosConst.DEFAULT_GROUP, 3000); + metadataBeans = JSONUtil.toList(content, MetadataBean.class); + configService.addListener(NacosConst.META_DATA_ID, NacosConst.DEFAULT_GROUP, new Listener() { + @Override + public Executor getExecutor() { + return null; + } + + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo Schema Tables {}", configInfo); + metadataBeans = JSONUtil.toList(configInfo, MetadataBean.class); + } + }); + } catch (NacosException e) { + log.info("NacosException:{}", e); + } + } + + private void initFields() { + for (MetadataBean metadataBean : metadataBeans) { + List<String> tables = metadataBean.getTables(); + for (String table : tables) { + try { + ConfigService configService = NacosFactory.createConfigService(nacosConfig.getProperties(NacosConst.NAMESPACE_AVRO)); + String content = configService.getConfig(table, metadataBean.getGroup(), 3000); + addSchema(content, metadataBean); + configService.addListener(table, metadataBean.getGroup(), new Listener() { + @Override + public Executor getExecutor() { + return null; + } + + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo Schema Fields {}", configInfo); + updateSchema(configInfo); + } + }); + } catch (NacosException e) { + log.info("NacosException:{}", e); + } + } + } + } + + private BaseResult loadFields(String name) { + Schema schema = getSchemaByName(name); + if (StringUtil.isEmpty(schema)) { + return BaseResultGenerator.failure(HttpStatus.SC_NOT_FOUND, + ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "Not found, check url please"); + } + Map resultMap = (Map) JsonMapper.fromJsonString(String.valueOf(schema), Map.class); + Map schemaDoc = (Map) JsonMapper.fromJsonString(schema.getDoc(), Map.class); + if (StringUtil.isEmpty(schemaDoc) && + StringUtil.isNotEmpty(schema.getDoc())) { + log.error("{} schema's doc isn't jsonString and won't convert: {}", name, schema.getDoc()); + } + if (StringUtil.isNotEmpty(schemaDoc)) { + resultMap.put("doc", schemaDoc); + } + List<Map> fields = (List<Map>) resultMap.get("fields"); + Map mapDoc; + for (Map next : fields) { + mapDoc = (Map) JsonMapper.fromJsonString(String.valueOf(next.get("doc")), Map.class); + if (StringUtil.isNotEmpty(mapDoc)) { + next.put("doc", mapDoc); + } + if (StringUtil.isEmpty(mapDoc) && !StringUtil.isEmpty(next.get("doc"))) { + log.error("{} field's doc isn't jsonString and won't convert: {}", next.get("name"), next.get("doc")); + } + } + return BaseResultGenerator.success("ok", resultMap); + } + + /** + * @param name + * @Description: 从Nacos配置中心获取Tables + * @Author: liuyongqiang + * @Date: 2021/3/11 18:17 + * @return: com.mesalab.common.base.BaseResult + **/ + private BaseResult loadTables(String name) { + Map<String, Object> date = new LinkedHashMap<>(); + List<String> tables = new ArrayList<>(); + boolean existName = false; + for (MetadataBean metadataBean : metadataBeans) { + String namespace = metadataBean.getNamespace(); + if (namespace.equalsIgnoreCase(name)) { + existName = true; + tables.addAll(metadataBean.getTables()); + } + } + if (existName) { + date.put("type", "enum"); + date.put("name", name); + date.put("symbols", tables); + } else { + return BaseResultGenerator.failure(HttpStatus.SC_NOT_FOUND, + ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "Not found, check url please"); + } + return BaseResultGenerator.success("ok", date); + } + + private Schema getSchemaByName(String tableName) { + Map<String, Schema> schemas = getAllSchema(); + return schemas.get(tableName); + } + + private void addSchema(String content, MetadataBean metadataBean) { + if (StrUtil.isBlank(content)) return; + Schema schema = new Schema.Parser().parse(content); + allSchemaMap.put(schema.getName(), schema); + + if (NacosConst.CLICKHOUSE_GROUP.equals(metadataBean.getGroup())) { + clickHouseSchemaMap.put(schema.getName(), schema); + } else if (NacosConst.DRUID_GROUP.equals(metadataBean.getGroup())) { + druidSchemaMap.put(schema.getName(), schema); + } + } + + private void updateSchema(String configInfo) { + if (StrUtil.isBlank(configInfo)) return; + Schema schema = new Schema.Parser().parse(configInfo); + allSchemaMap.put(schema.getName(), schema); + Schema ckSchema = clickHouseSchemaMap.get(schema.getName()); + if (Objects.nonNull(ckSchema)) { + clickHouseSchemaMap.put(schema.getName(), schema); + } + Schema druidSchema = druidSchemaMap.get(schema.getName()); + if (Objects.nonNull(druidSchema)) { + druidSchemaMap.put(schema.getName(), schema); + } + } + +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java index e5ca182e..477660be 100644 --- a/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java +++ b/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java @@ -36,6 +36,7 @@ import org.springframework.cglib.beans.BeanMap; import org.springframework.stereotype.Service; import org.springframework.util.ReflectionUtils; +import javax.annotation.Resource; import java.lang.reflect.AccessibleObject; import java.lang.reflect.Constructor; import java.lang.reflect.Field; @@ -50,7 +51,7 @@ public class SystemServiceImpl implements SystemService { private JobAdminHttpSource jobAdminHttpSource; @Autowired private HttpClientService httpClientService; - @Autowired + @Resource(name = "nacosMetadataService") private MetadataService metadataService; @Autowired HttpConfig httpConfig; |