diff options
| author | 窦凤虎 <[email protected]> | 2024-07-15 14:14:04 +0000 |
|---|---|---|
| committer | 窦凤虎 <[email protected]> | 2024-07-15 14:14:04 +0000 |
| commit | 3e68ae6f67eb16174807d83f1eae2a203ad235f0 (patch) | |
| tree | 21eeb6684b826885824993a0cb4a2bf7cf32405b | |
| parent | 1f8a10c9d5f1f839d22710aba953aadaba66494b (diff) | |
| parent | ea9ae304972a683883515679c243dfc9c1b44a8d (diff) | |
Merge branch 'develop' into 'master'master
Revert "[Fix][schema] 修正dos event schema field_discovery_metric session计算方式(TSG-21222)"
See merge request galaxy/platform/galaxy-qgw-service!40
389 files changed, 33375 insertions, 26631 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79c3c9be..8a8c2eef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,9 @@ +image: 192.168.40.153:8082/common/maven:3.8.1-openjdk-11-slim + +variables: + MAVEN_CLI_OPTS: --batch-mode --errors --show-version + SONAR_HOST_URL: http://192.168.40.153:9900 + SONAR_PROJECT_KEY: galaxy-qgw-service stages: - build @@ -13,7 +19,8 @@ build: only: - tags tags: - - galaxy + - docker-runner + clean: stage: clean @@ -25,15 +32,20 @@ clean: only: - tags tags: - - galaxy + - docker-runner + deploy: stage: deploy script: - echo 'exec mvn package & docker build' - echo 'tag:' $CI_COMMIT_TAG + - |- + mvn $MAVEN_CLI_OPTS clean verify sonar:sonar -Dsonar.projectKey=$SONAR_PROJECT_KEY \ + -Dsonar.host.url=$SONAR_HOST_URL -Dsonar.login=$SONAR_LOGIN_TOKEN \ + -Dsonar.qualitygate.wait=true -Dsonar.qualitygate.timeout=300 - mvn clean package -Dmaven.test.skip=true docker:build -DdockerImageTags=$CI_COMMIT_TAG when: on_success only: - tags tags: - - galaxy
\ No newline at end of file + - docker-runner
\ No newline at end of file @@ -1,21 +1,3 @@ MIT License -Copyright (c) 2018 dazzlzy - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +Copyright (c) 2018 @@ -65,9 +65,9 @@ 4. 执行./build-deploy.sh 完成部署 -5. 启停/home/tsg/galaxy/galaxy-qgw-service/bin start.sh|stop.sh +5. 启停/opt/galaxy-qgw-service/bin start.sh|stop.sh -6. 查看日志：tail -f /home/tsg/galaxy/galaxy-qgw-service/logs/galaxy-qgw-service.log +6. 查看日志：tail -f /opt/galaxy-qgw-service/logs/galaxy-qgw-service.log 7. 修改配置文件 @@ -85,17 +85,17 @@ 2. 修改镜像tag docker tag 79470009165e ceiec/galaxy-qgw-service:[version] -3. 部署容器 docker run -d --name galaxy-qgw-service -v /home/tsg/galaxy/galaxy-qgw-service/logs:/home/tsg/galaxy/galaxy-qgw-service/logs -p 9999:9999 --restart=always galaxy/galaxy-qgw-service:[version] +3. 部署容器 docker run -d --name galaxy-qgw-service -v /opt/galaxy-qgw-service/logs:/opt/galaxy-qgw-service/logs -p 9999:9999 --restart=always galaxy/galaxy-qgw-service:[version] 4. 启停 docker start|stop galaxy-qgw-service -5. 查看日志tail -f /home/tsg/galaxy/galaxy-qgw-service/logs/galaxy-qgw-service.log -6. 备份镜像文件 docker save 79470009165e > /home/tsg/galaxy/docker_image_galaxy-qgw-service_[version].tar +5. 查看日志tail -f /opt/galaxy-qgw-service/logs/galaxy-qgw-service.log +6. 备份镜像文件 docker save 79470009165e > /opt/docker_image_galaxy-qgw-service_[version].tar - GIT 提交规范 - [purpose][module name] Description (JIRA Issue ID) - - purpose : Fix, Feature, Improve, Docs - - module name : engine, cn, network, sql-dataset + - purpose : Fix, Feature, Improve, Refactor, Docs + - module name : query, dataset, database, util, troubleshooting, hos, admin, knowledge base file - 部署前提 diff --git a/benchmark/live_traffic_chart/livecharts-10min.txt b/benchmark/live_traffic_chart/livecharts-10min.txt index 42e4300d..2f5669e4 100644 --- a/benchmark/live_traffic_chart/livecharts-10min.txt +++ b/benchmark/live_traffic_chart/livecharts-10min.txt @@ -10,7 +10,7 @@ "result_bytes": 1329773, "result_rows": 9224 }, - "formatType": "json", + "output_mode": "json", "meta": [ { "name": "protocol_stack_id", diff --git a/benchmark/live_traffic_chart/livecharts-1h.txt b/benchmark/live_traffic_chart/livecharts-1h.txt index e7d4ed0f..ac88a229 100644 --- a/benchmark/live_traffic_chart/livecharts-1h.txt +++ b/benchmark/live_traffic_chart/livecharts-1h.txt @@ -10,7 +10,7 @@ "result_bytes": 2094352, "result_rows": 14371 }, - "formatType": "json", + "output_mode": "json", "meta": [ { "name": "protocol_stack_id", diff --git a/benchmark/live_traffic_chart/livecharts-30min.txt b/benchmark/live_traffic_chart/livecharts-30min.txt index 7d991bbd..898e114c 100644 --- a/benchmark/live_traffic_chart/livecharts-30min.txt +++ b/benchmark/live_traffic_chart/livecharts-30min.txt @@ -10,7 +10,7 @@ "result_bytes": 1870071, "result_rows": 12860 }, - "formatType": "json", + "output_mode": "json", "meta": [ { "name": "protocol_stack_id", diff --git a/benchmark/live_traffic_chart/livecharts-5min.txt b/benchmark/live_traffic_chart/livecharts-5min.txt index 2050f84f..ad100e8c 100644 --- a/benchmark/live_traffic_chart/livecharts-5min.txt +++ b/benchmark/live_traffic_chart/livecharts-5min.txt @@ -10,7 +10,7 @@ "result_bytes": 1027564, "result_rows": 7168 }, - "formatType": "json", + "output_mode": "json", "meta": [ { "name": "protocol_stack_id", diff --git a/config/application.yml b/config/application.yml index 333aa8fc..eec0d0a5 100644 --- a/config/application.yml +++ b/config/application.yml @@ -15,9 +15,9 @@ nacos: logging: config: ./config/log4j2.xml -#配置文件传输大小(上传文件大小:目前1个G) +#配置文件传输大小(上传文件大小:目前5个G) spring: servlet: multipart: - max-file-size: 1073741824 - max-request-size: 1073741824 + max-file-size: 5368709120 + max-request-size: 5368709120 diff --git a/config/flyway/cn/R__init_datasets.sql b/config/flyway/cn/R__init_datasets.sql index 1436abd5..29cb15bb 100644 --- a/config/flyway/cn/R__init_datasets.sql +++ b/config/flyway/cn/R__init_datasets.sql @@ -12,8 +12,8 @@ CREATE TABLE `dataset` ( `id` int(64) NOT NULL AUTO_INCREMENT, `identifier_name` varchar(128) NOT NULL, `category` varchar(128) DEFAULT NULL, - `execute_engine` varchar(16) NOT NULL DEFAULT 'qgw' COMMENT 'qgw, clickhouse, druid, hbase', - `type` varchar(16) NOT NULL COMMENT 'dsl,sql,long_polling', + `backend_engine` varchar(16) NOT NULL DEFAULT 'qgw' COMMENT 'qgw, clickhouse, druid, hbase', + `type` varchar(16) NOT NULL COMMENT 'dsl,sql', `template` text NOT NULL, `description` varchar(512) DEFAULT NULL, `generated_time` bigint DEFAULT NULL, diff --git a/config/flyway/cn/V23.12__init_saved_query_job.sql b/config/flyway/cn/V23.12__init_saved_query_job.sql index fdfff050..b0c7ca1d 100644 --- a/config/flyway/cn/V23.12__init_saved_query_job.sql +++ b/config/flyway/cn/V23.12__init_saved_query_job.sql @@ -4,8 +4,7 @@ SET FOREIGN_KEY_CHECKS = 0; -- ---------------------------- -- Table structure for saved_query_job -- ---------------------------- -DROP TABLE IF EXISTS `saved_query_job`; -CREATE TABLE `saved_query_job` ( +CREATE TABLE IF NOT EXISTS `saved_query_job` ( `job_id` varchar(128) NOT NULL COMMENT 'Job unique identifier', `query_sql` text NOT NULL COMMENT 'Query SQL', `state` varchar(16) NOT NULL COMMENT 'Enums: PENDING or RUNNING or DONE', diff --git a/config/flyway/tsg/R__init_datasets.sql b/config/flyway/tsg/R__init_datasets.sql index 4be08210..62342b2b 100644 --- a/config/flyway/tsg/R__init_datasets.sql +++ b/config/flyway/tsg/R__init_datasets.sql @@ -12,8 +12,8 @@ CREATE TABLE `dataset` ( `id` int(64) NOT NULL AUTO_INCREMENT, `identifier_name` varchar(128) NOT NULL, `category` varchar(128) DEFAULT NULL, - `execute_engine` varchar(16) NOT NULL DEFAULT 'qgw' COMMENT 'qgw, clickhouse, druid, hbase', - `type` varchar(16) NOT NULL COMMENT 'dsl,sql,long_polling', + `backend_engine` varchar(16) NOT NULL DEFAULT 'qgw' COMMENT 'qgw, clickhouse, druid, hbase', + `type` varchar(16) NOT NULL COMMENT 'dsl,sql', `template` text NOT NULL, `description` varchar(512) DEFAULT NULL, `generated_time` bigint DEFAULT NULL, @@ -40,136 +40,173 @@ DELIMITER ; -- Records of dataset -- ---------------------------- BEGIN; -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('current-network-throughput-trend', 'traffic_general_stat', 'qgw', 'sql', 'SELECT SUM( sum_in_bytes)* 8 / 15 AS avg_in_bits_per_sec, SUM( sum_out_bytes )* 8 / 15 AS avg_out_bits_per_sec, SUM( sum_in_bytes + sum_out_bytes )* 8 / 15 AS avg_bits_per_sec, SUM( sum_in_bytes )/ 15 AS avg_in_bytes_per_sec, SUM( sum_out_bytes )/ 15 AS avg_out_bytes_per_sec, SUM( sum_in_bytes + sum_out_bytes )/ 15 AS avg_bytes_per_sec, SUM( sum_in_pkts )/ 15 AS avg_in_pkts_per_sec, SUM( sum_out_pkts )/ 15 AS avg_out_pkts_per_sec, SUM( sum_in_pkts + sum_out_pkts )/ 15 AS avg_pkts_per_sec, SUM( sum_sessions )/ 15 AS avg_sessions_per_sec, SUM( max_active_sessions ) AS active_sessions FROM( SELECT device_id, vsys_id, SUM( in_bytes ) AS sum_in_bytes, SUM( out_bytes ) AS sum_out_bytes, SUM( in_pkts ) AS sum_in_pkts, SUM( out_pkts ) AS sum_out_pkts, SUM( sessions ) AS sum_sessions, max( active_sessions ) AS max_active_sessions FROM traffic_general_stat WHERE __time >= FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 30 ) AND __time < FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 15 ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY device_id, vsys_id )','System Overview-Traffic-Now. To calculate the current throughput, the metrics for each appliance are averaged over a 15-second interval. Additionally, the data ingestion latency is also set at 15 seconds.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('network-throughput-trend', 'traffic_general_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG( sum_in_bytes)* 8 / 5 AS avg_in_bits_per_sec, AVG( sum_out_bytes )* 8 / 5 AS avg_out_bits_per_sec, AVG( sum_bytes )* 8 / 5 AS avg_bits_per_sec, AVG( sum_in_bytes )/ 5 AS avg_in_bytes_per_sec, AVG( sum_out_bytes )/ 5 AS avg_out_bytes_per_sec, AVG( sum_bytes )/ 5 AS avg_bytes_per_sec, AVG( sum_in_pkts )/ 5 AS avg_in_pkts_per_sec, AVG( sum_out_pkts )/ 5 AS avg_out_pkts_per_sec, AVG( sum_pkts )/ 5 AS avg_pkts_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') AS stat_time, SUM( in_bytes ) AS sum_in_bytes, SUM( out_bytes ) AS sum_out_bytes, SUM( in_bytes + out_bytes ) AS sum_bytes, SUM( in_pkts ) AS sum_in_pkts, SUM( out_pkts ) AS sum_out_pkts, SUM( in_pkts + out_pkts ) AS sum_pkts FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}','The calculation of average network throughput is capable of supporting two types of metrics, namely "bytes" and "packets".'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('network-throughput-packet-per-second', 'traffic_general_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG( sum_in_pkts)/ 5 AS avg_in_pkts_per_sec, AVG( sum_out_pkts )/ 5 AS avg_out_pkts_per_sec, AVG( sum_pkts )/ 5 AS avg_pkts_per_sec, MAX( sum_in_pkts )/ 5 AS max_in_pkts_per_sec, MAX( sum_out_pkts )/ 5 AS max_out_pkts_per_sec, MAX( sum_pkts )/ 5 AS max_pkts_per_sec, MIN( sum_in_pkts )/ 5 AS min_in_pkts_per_sec, MIN( sum_out_pkts )/ 5 AS min_out_pkts_per_sec, MIN( sum_pkts )/ 5 AS min_pkts_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') AS stat_time, SUM( in_pkts ) AS sum_in_pkts, SUM( out_pkts ) AS sum_out_pkts, SUM( in_pkts + out_pkts ) AS sum_pkts FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}','System Overview-Traffic-AVG, MAX, MIN. To calculate network throughput, supports metric types "bytes" and "packets," and offers aggregation methods including "average"(avg), "maximum" (max), and "minimum" (min). Based on MAX and MIN values can reveal abnormal traffic patterns, such as peak and valley outliers.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('network-throughput-bit-per-second', 'traffic_general_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG( sum_in_bytes )* 8 / 5 AS avg_in_bits_per_sec, AVG( sum_out_bytes )* 8 / 5 AS avg_out_bits_per_sec, AVG( sum_bytes )* 8 / 5 AS avg_bits_per_sec, MAX( sum_in_bytes )* 8 / 5 AS max_in_bits_per_sec, MAX( sum_out_bytes )* 8 / 5 AS max_out_bits_per_sec, MAX( sum_bytes )* 8 / 5 AS max_bits_per_sec, MIN( sum_in_bytes )* 8 / 5 AS min_in_bits_per_sec, MIN( sum_out_bytes )* 8 / 5 AS min_out_bits_per_sec, MIN( sum_bytes )* 8 / 5 AS min_bits_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') AS stat_time, SUM( in_bytes ) AS sum_in_bytes, SUM( out_bytes ) AS sum_out_bytes, SUM( in_bytes + out_bytes ) AS sum_bytes FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( stat_time, ''${alignment_period}'', ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}','System Overview-Traffic-AVG, MAX, MIN. To calculate network throughput, supports metric types "bytes" and "packets," and offers aggregation methods including "average"(avg), "maximum" (max), and "minimum" (min). Based on MAX and MIN values can reveal abnormal traffic patterns, such as peak and valley outliers.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('network-throughput-byte-per-second', 'traffic_general_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG( sum_in_bytes)/ 5 AS avg_in_bytes_per_sec, AVG( sum_out_bytes )/ 5 AS avg_out_bytes_per_sec, AVG( sum_bytes )/ 5 AS avg_bytes_per_sec, MAX( sum_in_bytes )/ 5 AS max_in_bytes_per_sec, MAX( sum_out_bytes )/ 5 AS max_out_bytes_per_sec, MAX( sum_bytes )/ 5 AS max_bytes_per_sec, MIN( sum_in_bytes )/ 5 AS min_in_bytes_per_sec, MIN( sum_out_bytes )/ 5 AS min_out_bytes_per_sec, MIN( sum_bytes )/ 5 AS min_bytes_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') AS stat_time, SUM( in_bytes ) AS sum_in_bytes, SUM( out_bytes ) AS sum_out_bytes, SUM( in_bytes + out_bytes ) AS sum_bytes FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}','System Overview-Traffic-AVG, MAX, MIN. To calculate network throughput, supports metric types "bytes" and "packets," and offers aggregation methods including "average"(avg), "maximum" (max), and "minimum" (min). Based on MAX and MIN values can reveal abnormal traffic patterns, such as peak and valley outliers.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('new-sessions-per-second', 'traffic_general_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG( sum_sessions)/5 AS avg_sessions_per_sec , MAX( sum_sessions )/ 5 AS max_sessions_per_sec, MIN( sum_sessions )/ 5 AS min_sessions_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') AS stat_time, sum( sessions ) AS sum_sessions FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}','System Overview-New-AVG, MAX, MIN.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('active-sessions', 'traffic_general_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG( sum_active_sessions) AS avg_active_sessions, MAX( sum_active_sessions ) AS max_active_sessions, MIN( sum_active_sessions ) AS min_active_sessions FROM( SELECT stat_time, SUM( max_active_sessions ) sum_active_sessions FROM ( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') AS stat_time, device_id, vsys_id, MAX( active_sessions ) AS max_active_sessions FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S''), device_id, vsys_id ) GROUP BY stat_time ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('asymmetric-flows', 'traffic_general_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG( sum_bytes)* 8 / 5 AS avg_bits_per_sec, round( CASE WHEN sum( sum_closed_sessions ) = 0 THEN 0 ELSE SUM( sum_asymmetric_flows ) * 1.0 / SUM( sum_closed_sessions ) END, 4 ) * 100 AS percent_asymmetric_flows FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') AS stat_time, SUM( in_bytes ) AS sum_in_bytes, SUM( out_bytes ) AS sum_out_bytes, SUM( in_bytes + out_bytes ) AS sum_bytes, SUM( closed_sessions ) AS sum_closed_sessions, SUM( asymmetric_c2s_flows + asymmetric_s2c_flows ) AS sum_asymmetric_flows FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-statistics-metrics', 'traffic_general_stat', 'qgw', 'sql', 'SELECT COUNT(DISTINCT(device_id)) AS device_num, SUM(sum_bytes) AS total_bytes_transferred, SUM(sum_pkts) AS total_packets_transferred, SUM(sum_sessions) AS total_new_sessions , SUM(sum_closed_sessions) AS total_closed_sessions, SUM(sum_sessions)/ ${timestampdiff_second} AS avg_new_sessions_per_second, SUM(sum_bytes)* 8 / ${timestampdiff_second} AS avg_bits_per_second, SUM(sum_pkts)/ ${timestampdiff_second} AS avg_packets_per_second, SUM(avg_active_sessions) AS avg_active_sessions, round(CASE WHEN SUM(sum_closed_sessions) = 0 THEN 0 ELSE SUM(sum_asymmetric_flows) * 1.0 / SUM(sum_closed_sessions) END, 4) * 100 AS percent_asymmetric_flows FROM( SELECT device_id, vsys_id, SUM(in_bytes + out_bytes) AS sum_bytes, SUM(in_pkts + out_pkts) AS sum_pkts, SUM(sessions) AS sum_sessions, SUM(closed_sessions) AS sum_closed_sessions, AVG(active_sessions) AS avg_active_sessions, SUM(asymmetric_c2s_flows + asymmetric_s2c_flows) AS sum_asymmetric_flows FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time <''${end}'' AND vsys_id IN ( ${vsys_id}) AND ( ${filter} ) GROUP BY device_id, vsys_id )','predefined dataset for the report.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-summary-throughput', 'traffic_general_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time , AVG(sum_in_bytes)* 8 / 5 AS avg_in_bits_per_sec, AVG(sum_out_bytes)* 8 / 5 AS avg_out_bits_per_sec, AVG(sum_bytes)* 8 / 5 AS avg_bits_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') AS stat_time, SUM(in_bytes) AS sum_in_bytes, SUM(out_bytes) AS sum_out_bytes, SUM(in_bytes + out_bytes) AS sum_bytes FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id}) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}','predefined dataset for the report.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-summary-new-sessions-per-second', 'traffic_general_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG(sum_sessions)/ 5 AS avg_sessions_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'') AS stat_time, SUM(sessions) AS sum_sessions FROM traffic_general_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id}) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT5S'')) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}','predefined dataset for the report.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-summary', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{ "name": "application-and-protocol-summary", "granularity": "${alignment_period}", "filter": "vsys_id in(${vsys_id}) AND (${filter})", "intervals": ["${start}/${end}"] }','Application and Protocol Summary'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-tree-composition', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{ "name": "application-and-protocol-tree-composition", "filter": "vsys_id in(${vsys_id}) AND (${filter})", "intervals": ["${start}/${end}"] }','application and protocol tree composition'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-tree-throughput', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{ "name": "application-and-protocol-tree-throughput", "granularity": "${alignment_period}", "filter": "vsys_id in (${vsys_id}) AND (protocol_stack_id = ''${protocol_stack_id}'' OR ( protocol_stack_id LIKE ''${protocol_stack_id}.%'' AND NOT CONTAINS_STRING(REPLACE(protocol_stack_id, ''${protocol_stack_id}.'', ''''), ''.''))) AND (${filter})", "intervals": ["${start}/${end}"] }','application and protocol tree throughput'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-top-apps', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{ "name": "application-and-protocol-top-apps", "filter": "vsys_id in(${vsys_id}) AND (${filter})", "intervals": ["${start}/${end}"], "limit": ${limit} }','top apps'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-app-related-internal-ips', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{ "name": "application-and-protocol-app-related-internal-ips", "filter": "vsys_id in(${vsys_id}) AND app_name in (${app_name}) AND (${filter})", "intervals": ["${start}/${end}"], "limit": ${limit} }','app-related internal ips'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-app-throughput', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{ "name": "application-and-protocol-app-throughput", "granularity": "${alignment_period}", "filter": "vsys_id in(${vsys_id}) AND app_name in (${app_name}) AND (${filter})", "intervals": ["${start}/${end}"], "limit": ${limit} }','app-throughput'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-app-summary', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{ "name": "application-and-protocol-app-summary", "filter": "vsys_id in(${vsys_id}) AND app_name in (${app_name}) AND (${filter})", "intervals": ["${start}/${end}"] }','app summary'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('application-usage', 'traffic_application_protocol_stat', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, app_name, AVG( sum_in_bytes)* 8 / 15 AS avg_in_bits_per_sec, AVG( sum_out_bytes )* 8 / 15 AS avg_out_bits_per_sec, AVG( sum_bytes )* 8 / 15 AS avg_bits_per_sec, AVG( sum_in_pkts )/ 15 AS avg_in_pkts_per_sec, AVG( sum_out_pkts )/ 15 AS avg_out_pkts_per_sec, AVG( sum_pkts )/ 15 AS avg_pkts_per_sec, AVG( sum_sessions )/ 15 AS avg_sessions_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT15S'') AS stat_time, app_name, SUM( sessions ) AS sum_sessions, SUM( in_bytes ) AS sum_in_bytes, SUM( out_bytes ) AS sum_out_bytes, SUM( in_bytes + out_bytes ) AS sum_bytes, SUM( in_pkts ) AS sum_in_pkts, SUM( out_pkts ) AS sum_out_pkts, SUM( in_pkts + out_pkts ) AS sum_pkts FROM application_protocol_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) AND app_name IS NOT NULL GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT15S''), app_name ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')), app_name ORDER BY stat_time ASC LIMIT ${limit}','Real-time traffic sessions, bytes, and packets of favorite applications over a selected time frame.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('object-statistics-list', 'object_statistics', 'qgw', 'sql', 'SELECT object_id, SUM( in_bytes) AS in_bytes, SUM( out_bytes ) AS out_bytes, SUM( bytes ) AS bytes, SUM( new_in_sessions ) AS new_in_sessions, SUM( new_out_sessions ) AS new_out_sessions, SUM( sessions ) AS sessions FROM object_statistics WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND object_id IN ( ${object_id} ) AND ( ${filter} ) GROUP BY object_id ORDER BY ${order_by} LIMIT ${limit}','A list of objects sorted by ${metric}. The ${metric} variable includes [ incoming_bytes| outgoing_bytes| bytes| new_incoming_sessions| new_outgoing_sessions| sessions].'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('object-statistics-summary', 'object_statistics', 'qgw', 'sql', 'SELECT SUM( in_bytes) AS in_bytes, SUM( out_bytes ) AS out_bytes, SUM( bytes ) AS bytes, SUM( new_in_sessions ) AS new_in_sessions, SUM( new_out_sessions ) AS new_out_sessions, SUM( sessions ) AS sessions FROM object_statistics WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND object_id IN (${object_id}) AND item_id IN (${item_id}) AND ( ${filter} )','Aggregate traffic sessions and bytes based on the objects.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('object-statistics-item-list', 'object_statistics', 'qgw', 'sql', 'SELECT item_id, SUM( in_bytes) AS in_bytes, SUM( out_bytes ) AS out_bytes, SUM( bytes ) AS bytes, SUM( new_in_sessions ) AS new_in_sessions, SUM( new_out_sessions ) AS new_out_sessions, SUM( sessions ) AS sessions FROM object_statistics WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND object_id IN (${object_id}) AND item_id IN (${item_id}) AND item_id > 0 AND ( ${filter} ) GROUP BY item_id ORDER BY ${order_by} LIMIT ${limit}','A list of items sorted by ${metric} with specific Object ID(${filters}). The ${metric} variable includes [ incoming_bytes| outgoing_bytes| bytes| new_incoming_sessions| new_outgoing_sessions| sessions].'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('object-statistics-trend', 'object_statistics', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, SUM( in_bytes) AS in_bytes, AVG( in_bytes )* 8 / 15 AS avg_in_bits_per_sec, SUM( out_bytes ) AS out_bytes, AVG( out_bytes )* 8 / 15 AS avg_out_bits_per_sec, SUM( bytes ) AS bytes, AVG( bytes )* 8 / 15 AS avg_bits_per_sec, SUM( new_in_sessions ) AS new_in_sessions, AVG( new_in_sessions )/ 15 AS avg_new_in_sessions_per_sec, SUM( new_out_sessions ) AS new_out_sessions, AVG( new_out_sessions )/ 15 AS avg_new_out_sessions_per_sec, SUM( sessions ) AS sessions, AVG( sessions )/ 15 AS avg_sessions_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT15S'') AS stat_time, SUM( in_bytes ) AS in_bytes, SUM( out_bytes ) AS out_bytes, SUM( bytes ) AS bytes, SUM( new_in_sessions ) AS new_in_sessions, SUM( new_out_sessions ) AS new_out_sessions, SUM( sessions ) AS sessions FROM object_statistics WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND object_id IN (${object_id}) AND item_id IN (${item_id}) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT15S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}','The traffic trend by an object or item.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-dropped-packets', 'traffic_shaping_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''${alignment_period}'', ''zero'')) AS stat_time, rule_id, SUM( in_drop_pkts + out_drop_pkts) AS sum_dropped_packets FROM traffic_shaping_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''${alignment_period}'', ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}','The dropped packets for different traffic shaping rules over a selected time frame.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-throughput-by-rule-id', 'traffic_shaping_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, rule_id, AVG( sum_bytes)* 8 / 1 AS avg_bits_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') AS stat_time, rule_id, SUM( in_bytes + out_bytes ) AS sum_bytes FROM traffic_shaping_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S''), rule_id ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}','The throughput of traffic shaping rule by rule id over a selected time frame'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-profile-statistics', 'traffic_shaping_rule_hits', 'qgw', 'sql', 'SELECT profile_id, SUM( in_bytes + out_bytes) AS bytes, SUM( in_pkts + out_pkts ) AS packets, SUM( in_drop_pkts + out_drop_pkts ) AS drops, MAX( in_max_latency_us + out_max_latency_us ) AS max_latency_us, AVG( in_queue_len + out_queue_len ) AS avg_q, MAX( in_queue_len + out_queue_len ) AS max_q FROM traffic_shaping_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND profile_id IN ( ${profile_id} ) AND ( ${filter} ) GROUP BY profile_id','The statistics of specific shaping profiles.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-first-and-last-hit', 'traffic_shaping_rule_hits', 'qgw', 'sql', 'SELECT rule_id, DATE_FORMAT( MAX( __time), ''%Y-%m-%d %H:%i:%s'' ) AS last_hit, DATE_FORMAT( MIN( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS first_hit FROM traffic_shaping_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY rule_id','The time of the first and last hit for policy rules.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-transferred-bytes', 'traffic_shaping_rule_hits', 'qgw', 'sql', 'SELECT rule_id, SUM( in_bytes + out_bytes) AS total_bytes FROM traffic_shaping_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY rule_id','The number of transferred bytes of specific shaping rules.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-profile-first-and-last-hit', 'traffic_shaping_rule_hits', 'qgw', 'sql', 'SELECT profile_id, DATE_FORMAT( max( __time), ''%Y-%m-%d %H:%i:%s'' ) AS last_hit, DATE_FORMAT( min( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS first_hit, SUM( in_drop_pkts + out_drop_pkts ) AS drops FROM traffic_shaping_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND profile_id IN ( ${profile_id} ) AND ( ${filter} ) GROUP BY profile_id','The time of the first and last hit for shaping profiles.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-profile-current-throughput', 'traffic_shaping_rule_hits', 'qgw', 'sql', 'SELECT SUM( in_bytes)* 8 / 15 AS in_bps, SUM( out_bytes )* 8 / 15 AS out_bps, SUM( in_pkts )* 8 / 15 AS in_pps, SUM( out_pkts )* 8 / 15 AS out_pps, MAX( in_max_latency_us + out_max_latency_us ) AS max_latency_us, AVG( in_queue_len + out_queue_len ) AS avg_q, MAX( in_queue_len + out_queue_len ) AS max_q FROM traffic_shaping_rule_hits WHERE __time >= FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 30 ) AND __time < FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 15 ) AND vsys_id IN( ${vsys_id} ) AND profile_id IN ( ${profile_id} ) AND ( ${filter} )','To calculate the current shaping profile hit throughput, the metrics for each appliance are averaged over a 15-second interval. Additionally, the data ingestion latency is also set at 15 seconds.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-throughput', 'traffic_shaping_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time), ''${alignment_period}'', ''zero'' )) AS stat_time, SUM( in_bytes ) AS in_bytes, SUM( out_bytes ) AS out_bytes FROM traffic_shaping_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), ''${alignment_period}'', ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}','The throughput of traffic shaping rule over a selected time frame'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-profile-throughput', 'traffic_shaping_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time), ''${alignment_period}'', ''zero'' )) AS stat_time, SUM( in_bytes ) AS in_bytes, SUM( out_bytes ) AS out_bytes FROM traffic_shaping_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND profile_id IN ( ${profile_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), ''${alignment_period}'', ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}','The throughput of traffic shaping profile over a selected time frame'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('service-chaining-policy-rule-throughput-by-rule-id', 'service_chaining_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, rule_id, AVG( sum_bytes)* 8 / 1 AS avg_bits_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') AS stat_time, rule_id, SUM( sent_bytes + recv_bytes ) AS sum_bytes FROM service_chaining_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S''), rule_id ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}','The throughput of the service chaining rule over a selected time frame.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('service-chaining-policy-rule-transferred-bytes-trend', 'service_chaining_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time), ''${alignment_period}'', ''zero'' )) AS stat_time, SUM( sent_bytes ) AS sent_bytes, SUM( recv_bytes ) AS received_bytes FROM service_chaining_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), ''${alignment_period}'', ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('service-chaining-function-forwarder-profile-transferred-bytes-trend', 'service_chaining_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time), ''${alignment_period}'', ''zero'' )) AS stat_time, SUM( sent_bytes ) AS sent_bytes, SUM( recv_bytes ) AS received_bytes FROM service_chaining_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND sff_profile_id IN ( ${sff_profile_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), ''${alignment_period}'', ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('service-chaining-function-profile-transferred-bytes-trend', 'service_chaining_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time), ''${alignment_period}'', ''zero'' )) AS stat_time, SUM( sent_bytes ) AS sent_bytes, SUM( recv_bytes ) AS received_bytes FROM service_chaining_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND sf_profile_id IN ( ${sf_profile_id}) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), ''${alignment_period}'', ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('service-chaining-policy-rule-statistics', 'service_chaining_rule_hits', 'qgw', 'sql', 'SELECT rule_id, SUM(sent_bytes) as sent_bytes, SUM(recv_bytes) as received_bytes, SUM(sent_pkts) as sent_packets, SUM(recv_pkts) as received_packets FROM service_chaining_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id}) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY rule_id','The hit count of specific rules.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('service-chaining-function-forwarder-profile-statistics', 'service_chaining_rule_hits', 'qgw', 'sql', 'SELECT sff_profile_id, SUM( sent_bytes) AS sent_bytes, SUM( recv_bytes ) AS received_bytes, SUM( sent_pkts ) AS sent_packets, SUM( recv_pkts ) AS received_packets FROM service_chaining_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND sff_profile_id IN ( ${sff_profile_id} ) AND ( ${filter} ) GROUP BY sff_profile_id','The hit count of specific function forwarder profiles.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('service-chaining-function-profile-statistics', 'service_chaining_rule_hits', 'qgw', 'sql', 'SELECT sf_profile_id, SUM( sent_bytes) AS sent_bytes, SUM( recv_bytes ) AS received_bytes, SUM( sent_pkts ) AS sent_packets, SUM( recv_pkts ) AS received_packets FROM service_chaining_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND sf_profile_id IN ( ${sf_profile_id} ) AND ( ${filter} ) GROUP BY sf_profile_id','The hit count of specific function profiles.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('service-function-status-first-and-last-active-time', 'service_function_status', 'qgw', 'sql', 'SELECT sf_profile_id, sf_status, CASE WHEN last_active_time = 0 THEN '''' ELSE FROM_UNIXTIME(last_active_time) END AS last_active_time, CASE WHEN last_inactive_time = 0 THEN '''' ELSE FROM_UNIXTIME(last_inactive_time) END AS last_inactive_time FROM( SELECT sf_profile_id, LATEST(sf_status) as sf_status, MAX(CASE WHEN sf_status = 1 THEN UNIX_TIMESTAMP(__time) ELSE 0 END) as last_active_time, MAX(CASE WHEN sf_status = 0 THEN UNIX_TIMESTAMP(__time) ELSE 0 END) as last_inactive_time FROM service_function_status WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id}) AND sf_profile_id IN ( ${sf_profile_id} ) AND ( ${filter} ) GROUP BY sf_profile_id)',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-security-policy-rule', 'security_rule_hits', 'qgw', 'sql', 'SELECT rule_id AS rule_id, SUM( hit_count) AS hit_count, SUM( in_bytes + out_bytes ) AS bytes FROM security_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id ORDER BY ${metric} DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-hits-by-rule-id', 'security_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, rule_id, avg( sum_hit_count)/ 1 AS avg_hit_count_per_sec, avg( sum_bytes )* 8 / 1 AS avg_bits_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') AS stat_time, rule_id, sum( hit_count ) AS sum_hit_count, sum( in_bytes + out_bytes ) AS sum_bytes FROM security_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S''), rule_id ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}','The hit count and bytes of the favorite security policy rule over the selected time frame.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-hits-by-action', 'security_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time,(CASE WHEN action = 1 THEN ''Monitor'' WHEN action = 16 THEN ''Deny'' WHEN action = 96 THEN ''Allow'' WHEN action = 128 THEN ''Shunt'' ELSE concat( action) END ) AS action, AVG( sum_hit_count )/ 1 AS avg_hit_count_per_sec, AVG( sum_bytes )* 8 / 1 AS avg_bits_per_sec, AVG( sum_packets )/ 1 AS avg_pkts_per_sec, SUM( sum_hit_count ) AS total_hit_count, SUM( sum_bytes ) AS total_bytes, SUM( sum_packets ) AS total_packets FROM ( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') AS stat_time, action, SUM( hit_count ) AS sum_hit_count, SUM( in_bytes + out_bytes ) AS sum_bytes, SUM( in_pkts + out_pkts ) AS sum_packets FROM security_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S''), action ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')), action ORDER BY stat_time ASC LIMIT ${limit}','The security policy rule hits count, bytes, and packets of action over the selected time frame.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-security-policy-rule-with-action', 'security_rule_hits', 'qgw', 'sql', 'SELECT rule_id, action, SUM( hit_count) AS hit_count, SUM( in_bytes + out_bytes ) AS bytes FROM security_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id, action ORDER BY ${metric} DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-statistics', 'security_rule_hits', 'qgw', 'sql', 'SELECT rule_id, SUM(hit_count) as hits, SUM(in_bytes + out_bytes) as bytes FROM security_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND rule_id IN( ${rule_id}) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id','The statistics of specific rules.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-hits-trend', 'security_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time), ''${alignment_period}'', ''zero'' )) AS stat_time, SUM(hit_count) AS hits FROM security_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND rule_id IN(${rule_id}) AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), ''${alignment_period}'', ''zero'' )) LIMIT ${limit}','The trend of hit count by specific rule over a period of time.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-hits-first-and-last-hit', 'security_rule_hits', 'qgw', 'sql', 'SELECT rule_id, DATE_FORMAT(MIN(__time) ,''%Y-%m-%d %H:%i:%s'') AS first_hit, DATE_FORMAT(MAX(__time) ,''%Y-%m-%d %H:%i:%s'') AS last_hit FROM security_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND rule_id IN( ${rule_id}) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id','The time of the first and last hit for policy rules.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-current-throughput', 'security_rule_hits', 'qgw', 'sql', 'SELECT SUM(in_bytes)* 8 / 15 AS in_bps, SUM(out_bytes)* 8 / 15 AS out_bps, SUM(in_bytes + out_bytes)* 8 / 15 AS total_bps FROM security_rule_hits WHERE __time >= FROM_UNIXTIME(UNIX_TIMESTAMP(now())-30) AND __time < FROM_UNIXTIME(UNIX_TIMESTAMP(now())-15) AND vsys_id IN( ${vsys_id}) AND rule_id IN ( ${rule_id} ) AND ( ${filter} )','To calculate the current security hit throughput, the metrics for each appliance are averaged over a 15-second interval. Additionally, the data ingestion latency is also set at 15 seconds.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-proxy-policy-rule-with-manipulation-action', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT rule_id AS rule_id, sub_action AS sub_action, SUM( hit_count) AS hit_count, SUM( in_bytes + out_bytes ) AS bytes FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND action=48 AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id, sub_action ORDER BY ${metric} DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-by-rule-id', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, rule_id, avg( sum_hit_count)/ 1 AS avg_hit_count_per_sec, avg( sum_bytes )* 8 / 1 AS avg_bits_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') AS stat_time, rule_id, sum( hit_count ) AS sum_hit_count, sum( in_bytes + out_bytes ) AS sum_bytes FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S''), rule_id ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}','The hit count and bytes of the favorite proxy policy rule over the selected time frame.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-ssl-intercept-pinning-trend', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG( CASE WHEN pinning_status = ''0'' THEN sum_hit_count ELSE 0 END)/ 1 AS avg_not_pinning_hit_count_per_sec, AVG( CASE WHEN pinning_status = ''1'' THEN sum_hit_count ELSE 0 END )/ 1 AS avg_pinning_hit_count_per_sec, AVG( CASE WHEN pinning_status = ''2'' THEN sum_hit_count ELSE 0 END )/ 1 AS avg_maybe_pinning_hit_count_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') AS stat_time, pinning_status, SUM( hit_count ) AS sum_hit_count, SUM( in_bytes + out_bytes ) AS sum_bytes FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND action = 2 AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S''), pinning_status ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}','Monitor SSL Pinning over the selected time frame.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-ssl-intercept-pinning- statistics', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT( CASE WHEN pinning_status = ''0'' THEN ''not_pinning_num'' WHEN pinning_status = ''1'' THEN ''pinning_num'' WHEN pinning_status = ''2'' THEN ''maybe_pinning_num'' ELSE concat( pinning_status) END ) AS type, SUM( hit_count ) AS hits FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND action = 2 AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY pinning_status ORDER BY pinning_status','The statistics of Proxy SSL Intercept pinning.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-by-action', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time,(CASE WHEN action = 2 THEN ''Intercept'' WHEN action = 3 THEN ''No Intercept'' WHEN action = 48 THEN ''Manipulation'' ELSE concat( action) END ) AS action, AVG( sum_hit_count )/ 1 AS avg_hit_count_per_sec, SUM( sum_hit_count ) AS total_hit_count FROM ( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') AS stat_time, action AS action, SUM( hit_count ) AS sum_hit_count FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S''), action ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')), action ORDER BY stat_time ASC LIMIT ${limit}','The proxy Intercept policy hit the count of action over the selected time frame.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-manipulation-policy-rule-hits-by-action', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, action AS action, AVG( sum_hit_count)/ 1 AS avg_hit_count_per_sec, SUM( sum_hit_count ) AS total_hit_count FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') AS stat_time, sub_action AS action, SUM( hit_count ) AS sum_hit_count FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND action = 48 AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S''), sub_action ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')), action ORDER BY stat_time ASC LIMIT ${limit}','The proxy manipulation policy hit the count of action over the selected time frame.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-proxy-policy-rule', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT rule_id, SUM( hit_count) AS hit_count, SUM( in_bytes + out_bytes ) AS bytes FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id ORDER BY ${metric} DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-statistics', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT rule_id, SUM( hit_count) AS hits, SUM( in_bytes + out_bytes ) AS bytes FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND rule_id IN( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id LIMIT ${limit}','The statistics of specific rules.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-trend', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time), ''${alignment_period}'', ''zero'' )) AS stat_time, SUM( hit_count ) AS hits FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND rule_id IN( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), ''${alignment_period}'', ''zero'' )) LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-first-and-last-hit', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT rule_id, DATE_FORMAT( MIN( __time), ''%Y-%m-%d %H:%i:%s'' ) AS first_hit, DATE_FORMAT( MAX( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS last_hit FROM proxy_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND rule_id IN( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id','The time of the first and last hit for policy rules.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-current-throughput', 'proxy_rule_hits', 'qgw', 'sql', 'SELECT SUM( in_bytes)* 8 / 15 AS in_bps, SUM( out_bytes )* 8 / 15 AS out_bps, SUM( in_bytes + out_bytes )* 8 / 15 AS total_bps FROM proxy_rule_hits WHERE __time >= FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 30 ) AND __time < FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 15 ) AND vsys_id IN( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} )','To calculate the current policy hit throughput, the metrics for each appliance are averaged over a 15-second interval. Additionally, the data ingestion latency is also set at 15 seconds.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-hits-first-and-last-hit', 'statistics_rule_hits', 'qgw', 'sql', 'SELECT rule_id, DATE_FORMAT( min( __time), ''%Y-%m-%d %H:%i:%s'' ) AS first_hit, DATE_FORMAT( max( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS last_hit FROM statistics_rule_hits WHERE __time >= ''${start}'' AND __time <''${end}'' AND rule_id IN( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id','The time of the first and last hit for statistics policy rules.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-statistics', 'statistics_rule_hits', 'qgw', 'sql', 'SELECT rule_id, sum( hit_count) AS hits, sum( in_bytes + out_bytes ) AS bytes FROM statistics_rule_hits WHERE __time >= ''${start}'' AND __time <''${end}'' AND rule_id IN( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id','The statistics of specific rules.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-hits-trend', 'statistics_rule_hits', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time, AVG( sum_in_bytes)* 8 / 1 AS avg_in_bits_per_sec, AVG( sum_out_bytes )* 8 / 1 AS avg_out_bits_per_sec, AVG( sum_bytes )* 8 / 1 AS avg_bits_per_sec, SUM( sum_in_bytes ) AS total_in_bytes, SUM( sum_out_bytes ) AS total_out_bytes, SUM( sum_bytes ) AS total_bytes, AVG( sum_hit_count )/ 1 AS avg_hit_count_per_sec, SUM( sum_hit_count ) AS total_hit_count FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') AS stat_time, sum( in_bytes ) AS sum_in_bytes, sum( out_bytes ) AS sum_out_bytes, sum( in_bytes + out_bytes ) AS sum_bytes, sum( hit_count ) AS sum_hit_count FROM statistics_rule_hits WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id} ) AND rule_id IN (${rule_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT1S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-line-chart-template', 'statistics_rule', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) AS stat_time [[,AVG(${metric_in_bytes})*8/15 AS avg_in_bits_per_sec]] [[,AVG(${metric_out_bytes})*8/15 AS avg_out_bits_per_sec]] [[,AVG(${metric_bytes})*8/15 AS avg_bits_per_sec]] [[,AVG(${metric_new_c2s_flows})/15 AS avg_new_c2s_flows_per_sec]] [[,AVG(${metric_new_s2c_flows})/15 AS avg_new_s2c_flows_per_sec]] [[,AVG(${metric_new_in_sessions})/15 AS avg_new_in_sessions_per_sec]] [[,AVG(${metric_new_out_sessions})/15 AS avg_new_out_sessions_per_sec]] [[,AVG(${metric_sessions})/15 AS avg_sessions_per_sec]] [[,AVG(${metric_new_unestablished_sessions})/15 AS avg_new_unestablished_sessions_per_sec]] [[,AVG(${metric_syn_pkts})/15 AS avg_syn_pkts_per_sec]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_session_identifier_sketch}) AS active_sessions]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_client_ip_sketch}) AS unique_client_ips]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_server_ip_sketch}) AS unique_server_ips]] [[,APPROX_QUANTILE_HDR(${metric_in_latency_ms_sketch},0.5) AS median_tcp_in_latency_ms]] [[,APPROX_QUANTILE_HDR(${metric_in_latency_ms_sketch},0.95) AS p95th_tcp_in_latency_ms]] [[,APPROX_QUANTILE_HDR(${metric_in_latency_ms_sketch},0.99) AS p99th_tcp_in_latency_ms]] [[,APPROX_QUANTILE_HDR(${metric_out_latency_ms_sketch},0.5) AS median_tcp_out_latency_ms]] [[,APPROX_QUANTILE_HDR(${metric_out_latency_ms_sketch},0.95) AS p95th_tcp_out_latency_ms]] [[,APPROX_QUANTILE_HDR(${metric_out_latency_ms_sketch},0.99) AS p99th_tcp_out_latency_ms]] [[,APPROX_QUANTILE_HDR(${metric_latency_ms_sketch},0.5) AS median_tcp_latency_ms]] [[,APPROX_QUANTILE_HDR(${metric_latency_ms_sketch},0.95) AS p95th_tcp_latency_ms]] [[,APPROX_QUANTILE_HDR(${metric_latency_ms_sketch},0.99) AS p99th_tcp_latency_ms]] FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT15S'') AS stat_time [[,SUM(${metric_in_bytes}) AS in_bytes]] [[,SUM(${metric_out_bytes}) AS out_bytes]] [[,SUM(${metric_bytes}) AS bytes]] [[,SUM(${metric_new_c2s_flows}) AS new_c2s_flows]] [[,SUM(${metric_new_s2c_flows}) AS new_s2c_flows]] [[,SUM(${metric_new_in_sessions}) AS new_in_sessions]] [[,SUM(${metric_new_out_sessions}) AS new_out_sessions]] [[,SUM(${metric_sessions}) AS sessions]] [[,SUM(${metric_new_unestablished_sessions}) AS new_unestablished_sessions]] [[,SUM(${metric_syn_pkts}) AS syn_pkts]] [[,HLLD(${metric_session_identifier_sketch}) AS session_identifier_sketch]] [[,HLLD(${metric_client_ip_sketch}) AS client_ip_sketch]] [[,HLLD(${metric_server_ip_sketch}) AS server_ip_sketch]] [[,HDR_HISTOGRAM(${metric_in_latency_ms_sketch}) AS in_latency_ms_sketch]] [[,HDR_HISTOGRAM(${metric_out_latency_ms_sketch}) AS out_latency_ms_sketch]] [[,HDR_HISTOGRAM(${metric_latency_ms_sketch}) AS latency_ms_sketch]] FROM statistics_rule WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN ( ${vsys_id}) AND rule_id = ${rule_id} AND template_id = ${template_id} AND chart_id = ${chart_id} AND version = ${version} AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), ''PT15S'') ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, ''${alignment_period}'', ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-histogram-chart-template', 'statistics_rule', 'qgw', 'sql', 'SELECT [[PERCENTILES_HDR(${metric_in_latency_ms_sketch}) AS tcp_in_latency, QUANTILE_HDR(${metric_in_latency_ms_sketch}, 0.5) as tcp_in_latency_quantiles]] [[,PERCENTILES_HDR(${metric_out_latency_ms_sketch}) AS tcp_out_latency ,QUANTILE_HDR(${metric_out_latency_ms_sketch}, 0.5) as tcp_out_latency_quantiles]] [[,PERCENTILES_HDR(${metric_latency_ms_sketch}) AS tcp_latency ,QUANTILE_HDR(${metric_latency_ms_sketch}, 0.5) as tcp_latency_quantiles]] FROM statistics_rule WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id}) AND rule_id = ${rule_id} AND template_id = ${template_id} AND chart_id = ${chart_id} AND version = ${version} AND ( ${filter} ) LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-table-or-bar-chart-template', 'statistics_rule', 'qgw', 'sql', 'SELECT [[${dimension_client_ip} AS client_ip]] [[,${dimension_server_ip} AS server_ip]] [[,${dimension_server_fqdn} AS server_fqdn]] [[,${dimension_server_domain} AS server_domain]] [[,${dimension_application} AS application]] [[,${dimension_fqdn_category} AS fqdn_category]] [[,${dimension_client_ip_object} AS client_ip_object]] [[,${dimension_server_ip_object} AS server_ip_object]] [[,sum(${metric_in_bytes}) AS in_bytes]] [[,sum(${metric_out_bytes}) AS out_bytes]] [[,sum(${metric_bytes}) AS bytes]] [[,sum(${metric_new_c2s_flows}) AS new_c2s_flows]] [[,sum(${metric_new_s2c_flows}) AS new_s2c_flows]] [[,sum(${metric_new_in_sessions}) AS new_in_sessions]] [[,sum(${metric_new_out_sessions}) AS new_out_sessions]] [[,sum(${metric_sessions}) AS sessions]] [[,sum(${metric_new_unestablished_sessions}) AS new_unestablished_sessions]] [[,sum(${metric_syn_pkts}) AS syn_pkts]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_session_identifier_sketch}) AS active_sessions]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_client_ip_sketch}) AS unique_client_ips]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_server_ip_sketch}) AS unique_server_ips]] FROM statistics_rule WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id in(${vsys_id}) AND rule_id = ${rule_id} AND template_id = ${template_id} AND chart_id = ${chart_id} AND version = ${version} AND ( ${filter}) GROUP BY [[${dimension_client_ip}]] [[,${dimension_server_ip}]] [[,${dimension_server_fqdn}]] [[,${dimension_server_domain}]] [[,${dimension_application}]] [[,${dimension_fqdn_category}]] [[,${dimension_client_ip_object}]] [[,${dimension_server_ip_object}]] ORDER BY ${order_by} LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-client-ip', 'top_client_ips', 'qgw', 'sql', 'SELECT client_ip AS client_ip, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( out_bytes + in_bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( out_pkts + in_pkts ) AS packets FROM top_client_ips WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND metric = ''${metric}'' AND ( ${filter} ) GROUP BY client_ip ORDER BY ${metric} DESC LIMIT ${limit}','Aggregate traffic sessions, bytes, and packets based on the client''s IP address. The ${metric} variable includes three options: sessions, bytes, and packets.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-server-ip', 'top_server_ips', 'qgw', 'sql', 'SELECT server_ip AS server_ip, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( out_bytes + in_bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( out_pkts + in_pkts ) AS packets FROM top_server_ips WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) AND metric = ''${metric}'' GROUP BY server_ip ORDER BY ${metric} DESC LIMIT ${limit}','Aggregate traffic sessions, bytes, and packets based on the server IP address. The ${metric} variable includes three options: sessions, bytes, and packets.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-internal-ip', 'top_internal_ips', 'qgw', 'sql', 'SELECT internal_ip AS internal_ip, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( out_bytes + in_bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( out_pkts + in_pkts ) AS packets FROM top_internal_ips WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) AND metric = ''${metric}'' GROUP BY internal_ip ORDER BY ${metric} DESC LIMIT ${limit}','Aggregate traffic sessions, bytes, and packets based on the internal IP address. The ${metric} variable includes three options: sessions, bytes, and packets.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-external-ip', 'top_external_ips', 'qgw', 'sql', 'SELECT external_ip AS external_ip, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( out_bytes + in_bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( out_pkts + in_pkts ) AS packets FROM top_external_ips WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) AND metric = ''${metric}'' GROUP BY external_ip ORDER BY ${metric} DESC LIMIT ${limit}','Aggregate traffic sessions, bytes, and packets based on the external IP address. The ${metric} variable includes three options: sessions, bytes, and packets.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-subscriber-id', 'top_subscribers', 'qgw', 'sql', 'SELECT subscriber_id, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( out_bytes + in_bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( out_pkts + in_pkts ) AS packets FROM top_subscribers WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) AND metric = ''${metric}'' GROUP BY subscriber_id ORDER BY ${metric} DESC LIMIT ${limit}','Aggregate traffic sessions, bytes, and packets based on the subscribers. The ${metric} variable includes three options: sessions, bytes, and packets.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-server-domain', 'top_server_domains', 'qgw', 'sql', 'SELECT domain, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( out_bytes + in_bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( out_pkts + in_pkts ) AS packets FROM top_server_domains WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) AND metric = ''${metric}'' GROUP BY domain ORDER BY ${metric} DESC LIMIT ${limit}','Aggregate traffic sessions, bytes, and packets based on the server domains. The ${metric} variable includes three options: sessions, bytes, and packets.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-server-fqdn', 'top_server_fqdns', 'qgw', 'sql', 'SELECT fqdn, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( out_bytes + in_bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( out_pkts + in_pkts ) AS packets FROM top_server_fqdns WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) AND metric = ''${metric}'' GROUP BY fqdn ORDER BY ${metric} DESC LIMIT ${limit}','Aggregate traffic sessions, bytes, and packets based on the server FQDNs. The ${metric} variable includes three options: sessions, bytes, and packets.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('top-application', 'application_protocol_stat', 'qgw', 'sql', 'SELECT app_name, SUM( sessions) AS sessions, SUM( in_bytes ) AS in_bytes, SUM( out_bytes ) AS out_bytes, SUM( in_bytes + out_bytes ) AS bytes, SUM( in_pkts ) AS in_packets, SUM( out_pkts ) AS out_packets, SUM( in_pkts + out_pkts ) AS packets FROM application_protocol_stat WHERE __time >= ''${start}'' AND __time < ''${end}'' AND vsys_id IN( ${vsys_id} ) AND app_name IS NOT NULL AND ( ${filter} ) GROUP BY app_name ORDER BY ${metric} DESC LIMIT ${limit}','Aggregate traffic sessions, bytes, and packets based on the applications. The ${metric} variable includes three options: sessions, bytes, and packets.'); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('session-record-list', 'session_record', 'qgw', 'sql', 'SELECT ${columns} FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('session-record-count', 'session_record', 'qgw', 'sql', 'SELECT count(1) as count FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND ( ${filter})',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('session-record-timeline', 'session_record', 'qgw', 'sql', 'select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, ''${alignment_period}'', ''zero'')) AS stat_time, decoded_as as type, COUNT(1) as sessions, SUM(sent_bytes + received_bytes) as bytes, SUM(sent_pkts + received_pkts) as packets FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) GROUP BY stat_time, decoded_as',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('transaction-record-list', 'transaction_record', 'qgw', 'sql', 'SELECT ${columns} FROM transaction_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('security-event-list', 'security_event', 'qgw', 'sql', 'SELECT ${columns} FROM security_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('security-event-count', 'security_event', 'qgw', 'sql', 'SELECT count(1) as count FROM security_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND ( ${filter})',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('security-event-timeline', 'security_event', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, ''${alignment_period}'', ''zero'')) AS stat_time, security_action as type, COUNT(1) as sessions, SUM(sent_bytes + received_bytes) as bytes, SUM(sent_pkts + received_pkts) as packets FROM security_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND notEmpty(security_action) AND ( ${filter}) group by stat_time, security_action order by stat_time asc',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-event-intercept-list', 'session_record', 'qgw', 'sql', 'SELECT ${columns} FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND notEmpty(proxy_action) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-event-intercept-count', 'session_record', 'qgw', 'sql', 'SELECT count(1) as count FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND notEmpty(proxy_action) AND ( ${filter})',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-event-intercept-timeline', 'session_record', 'qgw', 'sql', 'select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, ''${alignment_period}'', ''zero'')) AS stat_time, decoded_as as type, COUNT(1) as sessions, SUM(sent_bytes + received_bytes) as bytes, SUM(sent_pkts + received_pkts) as packets FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND notEmpty(proxy_action) AND ( ${filter}) GROUP BY stat_time, decoded_as',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-event-manipulation-list', 'proxy_event', 'qgw', 'sql', 'SELECT ${columns} FROM proxy_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id in (${vsys_id}) AND ( ${filter} ) ORDER BY recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-event-manipulation-count', 'proxy_event', 'qgw', 'sql', 'SELECT COUNT(1) as count FROM proxy_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id in (${vsys_id}) AND ( ${filter} )',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('proxy-event-manipulation-timeline', 'proxy_event', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, ''${alignment_period}'', ''zero'')) AS stat_time, proxy_action as type, COUNT(1) as events FROM proxy_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN (${vsys_id}) AND notEmpty(proxy_action) AND ( ${filter} ) GROUP BY stat_time, proxy_action ORDER BY stat_time asc',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('radius-record-list', 'radius_record', 'qgw', 'sql', 'SELECT ${columns} FROM radius_record WHERE common_recv_time >= UNIX_TIMESTAMP(''${start}'') AND common_recv_time < UNIX_TIMESTAMP(''${end}'') AND common_vsys_id in(${vsys_id}) AND ( ${filter}) ORDER BY common_recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('radius-record-count', 'radius_record', 'qgw', 'sql', 'SELECT count(1) as count FROM radius_record WHERE common_recv_time >= UNIX_TIMESTAMP(''${start}'') AND common_recv_time < UNIX_TIMESTAMP(''${end}'') AND common_vsys_id in(${vsys_id}) AND ( ${filter})',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('radius-stat-summary', 'radius_onff_log', 'qgw', 'sql', 'SELECT COUNT(DISTINCT( framed_ip)) AS active_ip_num, SUM( acct_session_time ) AS online_duration FROM ( SELECT any( framed_ip ) AS framed_ip, MAX( acct_session_time ) AS acct_session_time FROM radius_onff_log WHERE account = ''${subscriber_id}'' AND event_timestamp >= UNIX_TIMESTAMP(''${start}'') AND event_timestamp < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY acct_session_id)',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('radius-subscriber-framed-ip', 'radius_onff_log', 'qgw', 'sql', 'SELECT DISTINCT( framed_ip) AS framed_ip FROM radius_onff_log WHERE account = ''${subscriber_id}'' AND event_timestamp >= UNIX_TIMESTAMP(''${start}'') AND event_timestamp < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN( ${vsys_id} ) AND ( ${filter} )',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('radius-subscriber-detail', 'radius_onff_log', 'qgw', 'sql', 'SELECT MAX(IF( acct_status_type = 1, event_timestamp, 0)) AS start_time, MAX(IF( acct_status_type = 2, event_timestamp, 0 )) AS end_time, any( framed_ip ) AS ip, MAX( acct_session_time ) AS online_duration FROM radius_onff_log WHERE account = ''${subscriber_id}'' AND event_timestamp >= UNIX_TIMESTAMP(''${start}'') AND event_timestamp < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN( ${vsys_id} ) AND ( ${filter} ) GROUP BY acct_session_id ORDER BY start_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('radius-record-timeline', 'radius_record', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(common_recv_time, ''${alignment_period}'', ''zero'')) AS stat_time, COUNT(1) as count FROM radius_record WHERE common_recv_time >= UNIX_TIMESTAMP(''${start}'') AND common_recv_time < UNIX_TIMESTAMP(''${end}'') AND common_vsys_id IN(${vsys_id}) AND ( ${filter}) GROUP BY stat_time ORDER BY stat_time asc',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('voip-record-list', 'voip_record', 'qgw', 'sql', 'SELECT ${columns} FROM voip_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id in(${vsys_id}) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('voip-record-count', 'voip_record', 'qgw', 'sql', 'SELECT count(1) as count FROM voip_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id in(${vsys_id}) AND ( ${filter})',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('voip-record-timeline', 'voip_record', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, ''${alignment_period}'', ''zero'')) AS stat_time, decoded_as as type, COUNT(1) as count FROM voip_record WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) GROUP BY stat_time,decoded_as ORDER BY stat_time asc',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('gtpc-record-list', 'gtpc_record', 'qgw', 'sql', 'SELECT ${columns} FROM gtpc_record WHERE common_recv_time >= UNIX_TIMESTAMP(''${start}'') AND common_recv_time < UNIX_TIMESTAMP(''${end}'') AND common_vsys_id in(${vsys_id}) AND ( ${filter}) ORDER BY common_recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('gtpc-record-count', 'gtpc_record', 'qgw', 'sql', 'SELECT count(1) as count FROM gtpc_record WHERE common_recv_time >= UNIX_TIMESTAMP(''${start}'') AND common_recv_time < UNIX_TIMESTAMP(''${end}'') AND common_vsys_id in(${vsys_id}) AND ( ${filter})',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('gtpc-record-timeline', 'gtpc_record', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(common_recv_time, ''${alignment_period}'', ''zero'')) AS stat_time, COUNT(1) as count FROM gtpc_record WHERE common_recv_time >= UNIX_TIMESTAMP(''${start}'') AND common_recv_time < UNIX_TIMESTAMP(''${end}'') AND common_vsys_id IN(${vsys_id}) AND ( ${filter}) GROUP BY stat_time ORDER BY stat_time asc',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-top-source-countries', 'dos_event', 'qgw', 'sql', 'SELECT arrayJoin(splitByString('','',source_country_list)) AS source_country, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND notEmpty(source_country_list) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY arrayJoin(splitByString('','',source_country_list)) ORDER BY count DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-top-destination-countries', 'dos_event', 'qgw', 'sql', 'SELECT destination_country, COUNT(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY destination_country ORDER BY count DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-top-victims', 'dos_event', 'qgw', 'sql', 'SELECT destination_ip, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY destination_ip ORDER BY count DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-attack-type', 'dos_event', 'qgw', 'sql', 'SELECT attack_type, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY attack_type ORDER BY attack_type LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-severity', 'dos_event', 'qgw', 'sql', 'SELECT severity, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY severity ORDER BY severity LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-destination-ip-distribution', 'dos_event', 'qgw', 'sql', 'SELECT destination_ip, IP_TO_GEO(destination_ip) AS destination_geo, any(destination_country) AS destination_country, groupUniqArray(arrayJoin(splitByString('','', source_country_list))) AS source_coutries, MAX(bit_rate) AS max_bit_rate, MAX(packet_rate) AS max_packet_rate, MAX(session_rate) AS max_session_rate, MIN(start_time) AS first_active_time, MAX(end_time) AS last_active_time, MAX_DURATION(end_time, 600) AS max_duration, groupUniqArray(attack_type) AS attack_type, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY destination_ip ORDER BY count DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-attack-connection', 'dos_event', 'qgw', 'sql', 'SELECT recv_time, destination_ip, IP_TO_GEO(destination_ip) AS destination_geo, destination_country, source_country_list, attack_type, severity, bit_rate, packet_rate, session_rate FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) ORDER BY recv_time ASC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-events-summary', 'dos_event', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, ''${alignment_period}'', ''zero'')) AS stat_time, attack_type AS type, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY stat_time, attack_type ORDER BY stat_time ASC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-events-attack-logs', 'dos_event', 'qgw', 'sql', 'SELECT log_id, profile_id, vsys_id, attack_type, source_country_list, source_ip_list, destination_ip, destination_country, severity, start_time, end_time, recv_time, conditions, bit_rate, packet_rate, session_rate FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) ORDER BY recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('dos-events-destination-ip-traffic-trend', 'dos_event', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), ''${alignment_period}'', ''zero'' )) AS stat_time, AVG(session_rate) AS session_rate, AVG(packet_rate) AS packet_rate, AVG(bit_rate) AS bit_rate FROM traffic_top_destination_ip_metrics_log WHERE __time >= ''${start}''AND __time < ''${end}''AND vsys_id IN (${vsys_id}) AND destination_ip = ''${destination_ip}''AND attack_type = ''${attack_type}''AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), ''${alignment_period}'', ''zero'' )) LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('packet-capture-policy-rule-hits', 'sys_packet_capture_event', 'qgw', 'sql', 'SELECT common_policy_id AS policy_id, count(1) AS hits, any(packet_url) AS packet_url FROM sys_packet_capture_event WHERE common_recv_time >=UNIX_TIMESTAMP(''${start}'') AND common_recv_time < UNIX_TIMESTAMP(''${end}'') AND common_policy_id IN(${policy_id}) AND pcap_storage_task_id IN (${task_id}) AND common_vsys_id IN (${vsys_id}) GROUP BY common_policy_id LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('packet-capture-sled-ip-statistics', 'sys_packet_capture_event', 'qgw', 'sql', 'SELECT common_sled_ip AS sled_ip, count(1) AS hits FROM sys_packet_capture_event WHERE common_recv_time >=UNIX_TIMESTAMP(''${start}'') AND common_recv_time < UNIX_TIMESTAMP(''${end}'') AND pcap_storage_task_id IN(${task_id}) AND common_vsys_id IN (${vsys_id}) GROUP BY common_sled_ip LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('packet-capture-event-list', 'sys_packet_capture_event', 'qgw', 'sql', 'SELECT toDateTime(common_recv_time) AS recv_time, common_client_ip AS client_ip, common_client_port AS client_port, common_server_ip AS server_ip, common_server_port AS server_port, common_l4_protocol AS l4_protocol, common_device_id AS device_id, common_sled_ip AS sled_ip,(common_c2s_byte_num + common_s2c_byte_num) bytes FROM sys_packet_capture_event WHERE common_recv_time>=UNIX_TIMESTAMP(''${start}'') AND common_recv_time < UNIX_TIMESTAMP(''${end}'') AND common_policy_id IN (${policy_id}) AND pcap_storage_task_id IN (${task_id}) AND common_vsys_id IN (${vsys_id}) ORDER BY common_recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('assessment-event-list', 'sys_packet_capture_event', 'qgw', 'sql', 'SELECT FROM_UNIXTIME(recv_time) AS recv_time, FROM_UNIXTIME(assessment_date) AS assessment_date, lot_number, file_name, features, assessment_type, size, file_checksum_sha, assessment_file FROM assessment_event WHERE recv_time >= UNIX_TIMESTAMP(''${start}'') AND recv_time < UNIX_TIMESTAMP(''${end}'') AND vsys_id IN(${vsys_id}) ORDER BY recv_time DESC LIMIT ${limit}',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-discovery', 'statistics', 'qgw', 'long_polling', '{ "query.name": "field_discovery", "query.data_source": "${source}", "custom.field_discovery.metric": "${metric}", "custom.field_discovery.metric.fn": "${fn}", "custom.field_discovery.fields": "${field_list}", "custom.field_discovery.filter": "${filter}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('report-dataset-statistics', 'statistics', 'qgw', 'long_polling', '{ "query.name":"saved_query", "query.data_source":"${source}", "custom.saved_query.sql":"{sql_statement}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-top-values', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT ${column_name}, count(*) as count FROM ${source} where ${filter} GROUP BY ${column_name} ORDER BY count DESC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-rare-values', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT ${column_name}, count(*) as count FROM ${source} where ${filter} GROUP BY ${column_name} ORDER BY count ASC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-avg-value-over-time', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT TIME_FLOOR_WITH_FILL(${timestamp_column}, ''${alignment_period}'', ''zero'') AS stat_time, avg(${column_name}) FROM ${source} WHERE ${filter} GROUP BY stat_time ORDER BY stat_time ASC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-max-value-over-time', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT TIME_FLOOR_WITH_FILL(${timestamp_column}, ''${alignment_period}'', ''zero'') AS stat_time, max(${column_name}) FROM ${source} WHERE ${filter} GROUP BY stat_time ORDER BY stat_time ASC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-min-value-over-time', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT TIME_FLOOR_WITH_FILL(${timestamp_column}, ''${alignment_period}'', ''zero'') AS stat_time, min(${column_name}) FROM ${source} WHERE ${filter} GROUP BY stat_time ORDER BY stat_time ASC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-median-value-over-time', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT TIME_FLOOR_WITH_FILL(${timestamp_column}, ''${alignment_period}'', ''zero'') AS stat_time, median(${column_name}) FROM ${source} WHERE ${filter} GROUP BY stat_time ORDER BY stat_time ASC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-p95-value-over-time', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT TIME_FLOOR_WITH_FILL(${timestamp_column}, ''${alignment_period}'', ''zero'') AS stat_time, QUANTILE(${column_name}, 0.95) FROM ${source} WHERE ${filter} GROUP BY stat_time ORDER BY stat_time ASC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-p99-value-over-time', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT TIME_FLOOR_WITH_FILL(${timestamp_column}, ''${alignment_period}'', ''zero'') AS stat_time, QUANTILE(${column_name}, 0.99) FROM ${source} WHERE ${filter} GROUP BY stat_time ORDER BY stat_time ASC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-top-frequent-elements-in-array', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT item as ${column_name}, sum(count) as count FROM(SELECT arrayJoin(items) as item,count FROM (SELECT ${column_name} as items, count(*) as count FROM ${source} as source where ${filter} and notEmpty(${column_name}) GROUP BY ${column_name})) GROUP BY item ORDER BY count DESC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-top-frequent-elements-in-string-of-array ', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT item as ${column_name}, sum(count) as count FROM(SELECT arrayJoin(items) as item,count FROM (SELECT splitByString(''${separator}'', ${column_name}) as items, count(*) as count FROM ${source} as source where ${filter} and notEmpty(${column_name}) GROUP BY ${column_name})) GROUP BY item ORDER BY count DESC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('field-statistics-top-frequent-elements-in-long-of-bit', 'statistics', 'qgw', 'long_polling', '{ "query.name":"statistics", "query.data_source":"${source}", "custom.statistics.sql":"SELECT item as ${column_name}, sum(count) as count FROM(SELECT arrayJoin(items) as item,count FROM (SELECT bitmaskToArray(${column_name}) as items, count(*) as count FROM ${source} as source where ${filter} GROUP BY ${column_name})) GROUP BY item ORDER BY count DESC LIMIT ${limit}" }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('real-time-data-analytics-subscriber-id-relate-ip', 'radius_record', 'qgw', 'dsl', '{ "name": "real-time-data-analytics-subscriber-id-relate-ip", "filter": "subscriber_id in(${subscriber_id}) AND vsys_id in (${vsys_id}) AND (${filter})", "intervals": ["${start}/${end}"], "limit": ${limit} }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('real-time-data-analytics-mobile-identity-relate-teid', 'gtpc_record', 'qgw', 'dsl', '{ "name": "real-time-data-analytics-mobile-identity-relate-teid", "filter": "vsys_id in(${vsys_id}) AND (${filter})", "intervals": ["${start}/${end}"], "limit": ${limit} }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('real-time-data-analytics-unstructured-data', 'unstructured_data', 'qgw', 'dsl', '{ "name": "real-time-data-analytics-unstructured-data", "filter": "file_type=''${file_type}'' AND data_source=''${data_source}'' AND(${filter})", "intervals": ["${start}/${end}"], "limit": ${limit} }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('ip-learning-fqdn-relate-ip', 'ip_learning', 'qgw', 'dsl', '{ "name": "ip-learning-fqdn-relate-ip", "filter": "vsys_id in(${vsys_id}) AND (${filter})", "intervals": ["${start}/${end}"], "limit": ${limit} }',null); -INSERT INTO `dataset` (`identifier_name`, `category`, `execute_engine`, `type`, `template`, `description`) VALUES ('ip-learning-active-ip', 'ip_learning', 'qgw', 'dsl', '{ "name": "ip-learning-active-ip", "filter": "vsys_id in(${vsys_id}) AND (${filter})", "intervals": ["${start}/${end}"], "order_by": "${order_by}" "limit": ${limit} }',null); - +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('current-network-throughput-trend', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT SUM( sum_in_bytes)* 8 / 5 AS avg_in_bits_per_sec, SUM( sum_out_bytes )* 8 / 5 AS avg_out_bits_per_sec, SUM( sum_in_bytes + sum_out_bytes )* 8 / 5 AS avg_bits_per_sec, SUM( sum_in_bytes )/ 5 AS avg_in_bytes_per_sec, SUM( sum_out_bytes )/ 5 AS avg_out_bytes_per_sec, SUM( sum_in_bytes + sum_out_bytes )/ 5 AS avg_bytes_per_sec, SUM( sum_in_pkts )/ 5 AS avg_in_pkts_per_sec, SUM( sum_out_pkts )/ 5 AS avg_out_pkts_per_sec, SUM( sum_in_pkts + sum_out_pkts )/ 5 AS avg_pkts_per_sec, SUM( sum_sessions )/ 5 AS avg_sessions_per_sec, SUM( max_active_sessions ) AS active_sessions FROM( SELECT device_id, vsys_id, SUM( in_bytes ) AS sum_in_bytes, SUM( out_bytes ) AS sum_out_bytes, SUM( in_pkts ) AS sum_in_pkts, SUM( out_pkts ) AS sum_out_pkts, SUM( sessions ) AS sum_sessions, max( active_sessions ) AS max_active_sessions FROM traffic_general_stat WHERE __time >= FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 15 ) AND __time < FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 10 ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY device_id, vsys_id )" }','System Overview-Traffic-Now. To calculate the current throughput, the metrics for each appliance are averaged over a 5-second interval. Additionally, the data ingestion latency is also set at 5 seconds.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('network-throughput-trend', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, RATE(in_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_in_bits_per_sec, RATE(out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''),1) * 8 AS avg_out_bits_per_sec, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''),1) * 8 AS avg_bits_per_sec, RATE(in_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''),1) AS avg_in_bytes_per_sec, RATE(out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''),1) AS avg_out_bytes_per_sec, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''),1) AS avg_bytes_per_sec, RATE(in_pkts, CHART_GRANULARITY(''${start_time}'', ''${end_time}''),1) AS avg_in_pkts_per_sec, RATE(out_pkts, CHART_GRANULARITY(''${start_time}'', ''${end_time}''),1) AS avg_out_pkts_per_sec, RATE(in_pkts + out_pkts, CHART_GRANULARITY(''${start_time}'', ''${end_time}''),1) AS avg_pkts_per_sec FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN(${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }','The calculation of average network throughput is capable of supporting two types of metrics, namely "bytes" and "packets".'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('network-throughput-packet-per-second', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, AVG( in_pkts_per_sec) AS avg_in_pkts_per_sec, AVG( out_pkts_per_sec ) AS avg_out_pkts_per_sec, AVG( pkts_per_sec ) AS avg_pkts_per_sec, MAX( in_pkts_per_sec ) AS max_in_pkts_per_sec, MAX( out_pkts_per_sec ) AS max_out_pkts_per_sec, MAX( pkts_per_sec ) AS max_pkts_per_sec, MIN( in_pkts_per_sec ) AS min_in_pkts_per_sec, MIN( out_pkts_per_sec ) AS min_out_pkts_per_sec, MIN( pkts_per_sec ) AS min_pkts_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) AS stat_time, RATE( in_pkts, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS in_pkts_per_sec, RATE( out_pkts, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS out_pkts_per_sec, RATE( in_pkts + out_pkts, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS pkts_per_sec FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }','System Overview-Traffic-AVG, MAX, MIN. To calculate network throughput, supports metric types "bytes" and "packets," and offers aggregation methods including "average"(avg), "maximum" (max), and "minimum" (min). Based on MAX and MIN values can reveal abnormal traffic patterns, such as peak and valley outliers.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('network-throughput-bit-per-second', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, AVG( in_bytes_per_sec )* 8 AS avg_in_bits_per_sec, AVG( out_bytes_per_sec )* 8 AS avg_out_bits_per_sec, AVG( bytes_per_sec )* 8 AS avg_bits_per_sec, MAX( in_bytes_per_sec )* 8 AS max_in_bits_per_sec, MAX( out_bytes_per_sec )* 8 AS max_out_bits_per_sec, MAX( bytes_per_sec )* 8 AS max_bits_per_sec, MIN( in_bytes_per_sec )* 8 AS min_in_bits_per_sec, MIN( out_bytes_per_sec )* 8 AS min_out_bits_per_sec, MIN( bytes_per_sec )* 8 AS min_bits_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) AS stat_time, RATE( in_bytes, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS in_bytes_per_sec, RATE( out_bytes, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS out_bytes_per_sec, RATE( in_bytes + out_bytes, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS bytes_per_sec FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}" }','System Overview-Traffic-AVG, MAX, MIN. To calculate network throughput, supports metric types "bytes" and "packets," and offers aggregation methods including "average"(avg), "maximum" (max), and "minimum" (min). Based on MAX and MIN values can reveal abnormal traffic patterns, such as peak and valley outliers.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('network-throughput-byte-per-second', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, AVG( in_bytes_per_sec) AS avg_in_bytes_per_sec, AVG( out_bytes_per_sec ) AS avg_out_bytes_per_sec, AVG( bytes_per_sec ) AS avg_bytes_per_sec, MAX( in_bytes_per_sec ) AS max_in_bytes_per_sec, MAX( out_bytes_per_sec ) AS max_out_bytes_per_sec, MAX( bytes_per_sec ) AS max_bytes_per_sec, MIN( in_bytes_per_sec ) AS min_in_bytes_per_sec, MIN( out_bytes_per_sec ) AS min_out_bytes_per_sec, MIN( bytes_per_sec ) AS min_bytes_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) AS stat_time, RATE( in_bytes, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS in_bytes_per_sec, RATE( out_bytes, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS out_bytes_per_sec, RATE( in_bytes + out_bytes, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS bytes_per_sec FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }','System Overview-Traffic-AVG, MAX, MIN. To calculate network throughput, supports metric types "bytes" and "packets," and offers aggregation methods including "average"(avg), "maximum" (max), and "minimum" (min). Based on MAX and MIN values can reveal abnormal traffic patterns, such as peak and valley outliers.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('new-sessions-per-second', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, AVG( sessions_per_sec) AS avg_sessions_per_sec, MAX( sessions_per_sec ) AS max_sessions_per_sec, MIN( sessions_per_sec ) AS min_sessions_per_sec FROM( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) AS stat_time, RATE( sessions, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS sessions_per_sec FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }','System Overview-New-AVG, MAX, MIN.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('active-sessions', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, AVG( active_sessions_per_sec) AS avg_active_sessions, MAX( active_sessions_per_sec ) AS max_active_sessions, MIN( active_sessions_per_sec ) AS min_active_sessions FROM( SELECT stat_time, RATE( max_active_sessions, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) active_sessions_per_sec FROM ( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) AS stat_time, device_id, vsys_id, MAX( active_sessions ) AS max_active_sessions FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')), device_id, vsys_id ) GROUP BY stat_time ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('asymmetric-flows', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'')) AS stat_time, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec, round( CASE WHEN SUM(closed_sessions) = 0 THEN 0 ELSE SUM( asymmetric_c2s_flows + asymmetric_s2c_flows) * 100.0 / SUM(closed_sessions) END,2) AS percent_asymmetric_flows FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN(${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-statistics-metrics', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT COUNT(DISTINCT(device_id)) AS device_num, SUM(sum_bytes) AS total_bytes_transferred, SUM(sum_pkts) AS total_packets_transferred, SUM(sum_sessions) AS total_new_sessions , SUM(sum_closed_sessions) AS total_closed_sessions, SUM(sum_sessions)/ ${timestampdiff_second} AS avg_new_sessions_per_second, SUM(sum_bytes)* 8 / ${timestampdiff_second} AS avg_bits_per_second, SUM(sum_pkts)/ ${timestampdiff_second} AS avg_packets_per_second, SUM(avg_active_sessions) AS avg_active_sessions, round(CASE WHEN SUM(sum_closed_sessions) = 0 THEN 0 ELSE SUM(sum_asymmetric_flows) * 100.0 / SUM(sum_closed_sessions) END, 2) AS percent_asymmetric_flows FROM( SELECT device_id, vsys_id, SUM(in_bytes + out_bytes) AS sum_bytes, SUM(in_pkts + out_pkts) AS sum_pkts, SUM(sessions) AS sum_sessions, SUM(closed_sessions) AS sum_closed_sessions, AVG(active_sessions) AS avg_active_sessions, SUM(asymmetric_c2s_flows + asymmetric_s2c_flows) AS sum_asymmetric_flows FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time <''${end_time}'' AND vsys_id IN ( ${vsys_id}) AND ( ${filter} ) GROUP BY device_id, vsys_id )" }','predefined dataset for the report.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-summary-throughput', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'')) AS stat_time , RATE(in_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_in_bits_per_sec, RATE(out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_out_bits_per_sec, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN(${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }','predefined dataset for the report.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-summary-new-sessions-per-second', 'traffic_general_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'')) AS stat_time, RATE(sessions, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_sessions_per_sec FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN(${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }','predefined dataset for the report.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-summary', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{"name": "application-and-protocol-summary", "filter": "vsys_id in (${vsys_id}) AND (${filter})", "intervals": ["${start_time}/${end_time}"] }','Application and Protocol Summary'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-tree-composition', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{"name": "application-and-protocol-tree-composition", "filter": "vsys_id in (${vsys_id}) AND (${filter})", "intervals": ["${start_time}/${end_time}"] }','application and protocol tree composition'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-tree-throughput', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{"name": "application-and-protocol-tree-throughput", "filter": "vsys_id in (${vsys_id}) AND (protocol_stack_id = ''${protocol_stack_id}'' OR ( protocol_stack_id LIKE ''${protocol_stack_id}.%'' AND NOT CONTAINS_STRING(REPLACE(protocol_stack_id, ''${protocol_stack_id}.'', ''''), ''.''))) AND (${filter})", "intervals": ["${start_time}/${end_time}"] }','application and protocol tree throughput'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-top-apps', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{"name": "application-and-protocol-top-apps", "filter": "vsys_id in (${vsys_id}) AND (${filter})", "intervals": ["${start_time}/${end_time}"], "limit": "${limit}" }','top apps'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-app-related-internal-ips', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{"name": "application-and-protocol-app-related-internal-ips", "filter": "vsys_id in (${vsys_id}) AND app in (''${app_name}'') AND (${filter})", "intervals": ["${start_time}/${end_time}"], "limit": "${limit}" }','app-related internal ips'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-app-throughput', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{"name": "application-and-protocol-app-throughput", "filter": "vsys_id in (${vsys_id}) AND app_name in (''${app_name}'') AND (${filter})", "intervals": ["${start_time}/${end_time}"], "limit": "${limit}" }','app-throughput'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('application-and-protocol-app-summary', 'traffic_application_protocol_stat', 'qgw', 'dsl', '{"name": "application-and-protocol-app-summary", "filter": "vsys_id in (${vsys_id}) AND app_name in (''${app_name}'') AND (${filter})", "intervals": ["${start_time}/${end_time}"] }','app summary'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('application-usage', 'traffic_application_protocol_stat', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, app_name, RATE(in_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_in_bits_per_sec, RATE(out_bytes,CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_out_bits_per_sec, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec, RATE(in_pkts, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_in_pkts_per_sec, RATE(out_pkts, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_out_pkts_per_sec, RATE(in_pkts + out_pkts, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_pkts_per_sec, RATE(sessions, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_sessions_per_sec FROM application_protocol_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) AND app_name IS NOT NULL GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')), app_name ORDER BY stat_time ASC LIMIT ${limit}" }','Real-time traffic sessions, bytes, and packets of favorite applications over a selected time frame.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('object-statistics-list', 'object_statistics', 'qgw', 'sql', '{ "statement": "SELECT object_id, SUM( in_bytes) AS in_bytes, SUM( out_bytes ) AS out_bytes, SUM( bytes ) AS bytes, SUM( new_in_sessions ) AS new_in_sessions, SUM( new_out_sessions ) AS new_out_sessions, SUM( sessions ) AS sessions FROM object_statistics WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND object_type IN ( ''${object_type}'' ) AND object_id IN ( ${object_id} ) AND ( ${filter} ) GROUP BY object_id ORDER BY ${order_by} LIMIT ${limit}" }','A list of objects sorted by ${metric}. The ${metric} variable includes [ incoming_bytes| outgoing_bytes| bytes| new_incoming_sessions| new_outgoing_sessions| sessions].'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('object-statistics-summary', 'object_statistics', 'qgw', 'sql', '{ "statement": "SELECT SUM( in_bytes) AS in_bytes, SUM( out_bytes ) AS out_bytes, SUM( bytes ) AS bytes, SUM( new_in_sessions ) AS new_in_sessions, SUM( new_out_sessions ) AS new_out_sessions, SUM( sessions ) AS sessions FROM object_statistics WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND object_type IN ( ''${object_type}'' ) AND object_id IN (${object_id}) AND item_id IN (${item_id}) AND ( ${filter} )" }','Aggregate traffic sessions and bytes based on the objects.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('object-statistics-item-list', 'object_statistics', 'qgw', 'sql', '{ "statement": "SELECT item_id, SUM( in_bytes) AS in_bytes, SUM( out_bytes ) AS out_bytes, SUM( bytes ) AS bytes, SUM( new_in_sessions ) AS new_in_sessions, SUM( new_out_sessions ) AS new_out_sessions, SUM( sessions ) AS sessions FROM object_statistics WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND object_type IN ( ''${object_type}'' ) AND object_id IN (${object_id}) AND item_id IN (${item_id}) AND item_id > 0 AND ( ${filter} ) GROUP BY item_id ORDER BY ${order_by} LIMIT ${limit}" }','A list of items sorted by ${metric} with specific Object ID(${filters}). The ${metric} variable includes [ incoming_bytes| outgoing_bytes| bytes| new_incoming_sessions| new_outgoing_sessions| sessions].'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('object-statistics-trend', 'object_statistics', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, SUM( in_bytes ) AS in_bytes, RATE(in_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_in_bits_per_sec, SUM( out_bytes ) AS out_bytes, RATE(out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_out_bits_per_sec, SUM( bytes ) AS bytes, RATE(bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec, SUM( new_in_sessions ) AS new_in_sessions, RATE(new_in_sessions, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_new_in_sessions_per_sec, SUM( new_out_sessions ) AS new_out_sessions, RATE(new_out_sessions, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_new_out_sessions_per_sec, SUM( sessions ) AS sessions, RATE(sessions, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_sessions_per_sec FROM object_statistics WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND object_type IN ( ''${object_type}'' ) AND object_id IN (${object_id}) AND item_id IN (${item_id}) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}"}','The traffic trend by an object or item.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-dropped-packets', 'traffic_shaping_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, rule_id, SUM( in_drop_pkts + out_drop_pkts ) AS sum_dropped_packets FROM traffic_shaping_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}" }','The dropped packets for different traffic shaping rules over a selected time frame.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-throughput-by-rule-id', 'traffic_shaping_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, rule_id, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec FROM traffic_shaping_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN (${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}" }','The throughput of traffic shaping rule by rule id over a selected time frame'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-profile-statistics', 'traffic_shaping_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT profile_id, SUM( in_bytes + out_bytes ) AS bytes, SUM( in_pkts + out_pkts ) AS packets, SUM( in_drop_pkts + out_drop_pkts ) AS drops, MAX( in_max_latency_us + out_max_latency_us ) AS max_latency_us, AVG( in_queue_len + out_queue_len ) AS avg_q, MAX( in_queue_len + out_queue_len ) AS max_q FROM traffic_shaping_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND profile_id IN ( ${profile_id} ) AND ( ${filter} ) GROUP BY profile_id" }','The statistics of specific shaping profiles.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-first-and-last-hit', 'traffic_shaping_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, DATE_FORMAT( MAX( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS last_hit, DATE_FORMAT( MIN( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS first_hit FROM traffic_shaping_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The time of the first and last hit for policy rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-transferred-bytes', 'traffic_shaping_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, SUM( in_bytes + out_bytes ) AS total_bytes FROM traffic_shaping_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The number of transferred bytes of specific shaping rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-profile-first-and-last-hit', 'traffic_shaping_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT profile_id, DATE_FORMAT( max( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS last_hit, DATE_FORMAT( min( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS first_hit, SUM( in_drop_pkts + out_drop_pkts ) AS drops FROM traffic_shaping_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND profile_id IN ( ${profile_id} ) AND ( ${filter} ) GROUP BY profile_id" }','The time of the first and last hit for shaping profiles.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-profile-current-throughput', 'traffic_shaping_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT SUM( in_bytes )* 8 / 5 AS in_bps, SUM( out_bytes )* 8 / 5 AS out_bps, SUM( in_pkts )* 8 / 5 AS in_pps, SUM( out_pkts )* 8 / 5 AS out_pps, MAX( in_max_latency_us + out_max_latency_us ) AS max_latency_us, AVG( in_queue_len + out_queue_len ) AS avg_q, MAX( in_queue_len + out_queue_len ) AS max_q FROM traffic_shaping_rule_hits WHERE __time >= FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 15 ) AND __time < FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 10 ) AND vsys_id IN ( ${vsys_id} ) AND profile_id IN ( ${profile_id} ) AND ( ${filter} )" }','To calculate the current shaping profile hit throughput, the metrics for each appliance are averaged over a 5-second interval. Additionally, the data ingestion latency is also set at 5 seconds.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-policy-rule-throughput', 'traffic_shaping_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) AS stat_time, SUM( in_bytes ) AS in_bytes, SUM( out_bytes ) AS out_bytes FROM traffic_shaping_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}" }','The throughput of traffic shaping rule over a selected time frame'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-shaping-profile-throughput', 'traffic_shaping_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) AS stat_time, SUM( in_bytes ) AS in_bytes, SUM( out_bytes ) AS out_bytes FROM traffic_shaping_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND profile_id IN ( ${profile_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}" }','The throughput of traffic shaping profile over a selected time frame'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('service-chaining-policy-rule-throughput-by-rule-id', 'service_chaining_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, rule_id, RATE(sent_bytes + recv_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec FROM service_chaining_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN (${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}" }','The throughput of the service chaining rule over a selected time frame.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('service-chaining-policy-rule-transferred-bytes-trend', 'service_chaining_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) AS stat_time, SUM( sent_bytes ) AS sent_bytes, SUM( recv_bytes ) AS received_bytes FROM service_chaining_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('service-chaining-function-forwarder-profile-transferred-bytes-trend', 'service_chaining_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) AS stat_time, SUM( sent_bytes ) AS sent_bytes, SUM( recv_bytes ) AS received_bytes FROM service_chaining_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND sff_profile_id IN ( ${sff_profile_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('service-chaining-function-profile-transferred-bytes-trend', 'service_chaining_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) AS stat_time, SUM( sent_bytes ) AS sent_bytes, SUM( recv_bytes ) AS received_bytes FROM service_chaining_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND sf_profile_id IN ( ${sf_profile_id}) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('service-chaining-policy-rule-statistics', 'service_chaining_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, SUM(sent_bytes) as sent_bytes, SUM(recv_bytes) as received_bytes, SUM(sent_pkts) as sent_packets, SUM(recv_pkts) as received_packets FROM service_chaining_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The hit count of specific rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('service-chaining-function-forwarder-profile-statistics', 'service_chaining_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT sff_profile_id, SUM( sent_bytes ) AS sent_bytes, SUM( recv_bytes ) AS received_bytes, SUM( sent_pkts ) AS sent_packets, SUM( recv_pkts ) AS received_packets FROM service_chaining_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND sff_profile_id IN ( ${sff_profile_id} ) AND ( ${filter} ) GROUP BY sff_profile_id" }','The hit count of specific function forwarder profiles.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('service-chaining-function-profile-statistics', 'service_chaining_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT sf_profile_id, SUM( sent_bytes ) AS sent_bytes, SUM( recv_bytes ) AS received_bytes, SUM( sent_pkts ) AS sent_packets, SUM( recv_pkts ) AS received_packets FROM service_chaining_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND sf_profile_id IN ( ${sf_profile_id} ) AND ( ${filter} ) GROUP BY sf_profile_id" }','The hit count of specific function profiles.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('service-function-status-first-and-last-active-time', 'service_function_status', 'qgw', 'sql', '{ "statement": "SELECT sf_profile_id, sf_status, CASE WHEN last_active_time = 0 THEN '''' ELSE FROM_UNIXTIME(last_active_time) END AS last_active_time, CASE WHEN last_inactive_time = 0 THEN '''' ELSE FROM_UNIXTIME(last_inactive_time) END AS last_inactive_time FROM (SELECT sf_profile_id, LATEST(sf_status) as sf_status, MAX(CASE WHEN sf_status = 1 THEN UNIX_TIMESTAMP(__time) ELSE 0 END) as last_active_time, MAX(CASE WHEN sf_status = 0 THEN UNIX_TIMESTAMP(__time) ELSE 0 END) as last_inactive_time FROM service_function_status WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND sf_profile_id IN ( ${sf_profile_id} ) AND ( ${filter} ) GROUP BY sf_profile_id)" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-security-policy-rule', 'security_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id AS rule_id, SUM( hit_count ) AS hit_count, SUM( in_bytes + out_bytes ) AS bytes FROM security_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id ORDER BY ${metric} DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-hits-by-rule-id', 'security_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, rule_id, RATE(hit_count, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_hit_count_per_sec, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec FROM security_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN (${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}" }','The hit count and bytes of the favorite security policy rule over the selected time frame.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-hits-by-action', 'security_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, (CASE WHEN action = 1 THEN ''Monitor'' WHEN action = 16 THEN ''Deny'' WHEN action = 96 THEN ''Allow'' WHEN action = 128 THEN ''Shunt'' ELSE concat(action) END ) AS action, RATE(hit_count, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_hit_count_per_sec, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec, RATE(in_pkts + out_pkts, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_pkts_per_sec, SUM(hit_count) AS total_hit_count, SUM( in_bytes + out_bytes ) AS total_bytes, SUM( in_pkts + out_pkts ) AS total_packets FROM security_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN (${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')), action ORDER BY stat_time ASC LIMIT ${limit}" }','The security policy rule hits count, bytes, and packets of action over the selected time frame.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-security-policy-rule-with-action', 'security_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, action, SUM( hit_count ) AS hit_count, SUM( in_bytes + out_bytes ) AS bytes FROM security_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id, action ORDER BY ${metric} DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-statistics', 'security_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, SUM(hit_count) as hits, SUM(in_bytes + out_bytes) as bytes FROM security_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The statistics of specific rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-hits-trend', 'security_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) AS stat_time, SUM(hit_count) AS hits FROM security_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND rule_id IN (${rule_id}) AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) LIMIT ${limit}" }','The trend of hit count by specific rule over a period of time.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-hits-first-and-last-hit', 'security_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, DATE_FORMAT(MIN(__time) ,''%Y-%m-%d %H:%i:%s'') AS first_hit, DATE_FORMAT(MAX(__time) ,''%Y-%m-%d %H:%i:%s'') AS last_hit FROM security_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The time of the first and last hit for policy rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('security-policy-rule-current-throughput', 'security_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT SUM(in_bytes)* 8 / 5 AS in_bps, SUM(out_bytes)* 8 / 5 AS out_bps, SUM(in_bytes + out_bytes)* 8 / 5 AS total_bps FROM security_rule_hits WHERE __time >= FROM_UNIXTIME(UNIX_TIMESTAMP(now())-15) AND __time < FROM_UNIXTIME(UNIX_TIMESTAMP(now())-10) AND vsys_id IN ( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} )" }','To calculate the current security hit throughput, the metrics for each appliance are averaged over a 5-second interval. Additionally, the data ingestion latency is also set at 5 seconds.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('monitor-policy-rule-statistics', 'monitor_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, SUM(hit_count) as hits, SUM(in_bytes + out_bytes) as bytes FROM monitor_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The statistics of specific rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('monitor-policy-rule-hits-trend', 'monitor_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) AS stat_time, SUM(hit_count) AS hits FROM monitor_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND rule_id IN (${rule_id}) AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) LIMIT ${limit}" }','The trend of hit count by specific rule over some time.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('monitor-policy-rule-hits-first-and-last-hit', 'monitor_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, DATE_FORMAT(MIN(__time) ,''%Y-%m-%d %H:%i:%s'') AS first_hit, DATE_FORMAT(MAX(__time) ,''%Y-%m-%d %H:%i:%s'') AS last_hit FROM monitor_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The time of the first and last hit for policy rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('monitor-policy-rule-current-throughput', 'monitor_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT SUM(in_bytes)* 8 / 5 AS in_bps, SUM(out_bytes)* 8 / 5 AS out_bps, SUM(in_bytes + out_bytes)* 8 / 5 AS total_bps FROM monitor_rule_hits WHERE __time >= FROM_UNIXTIME(UNIX_TIMESTAMP(now())-15) AND __time < FROM_UNIXTIME(UNIX_TIMESTAMP(now())-10) AND vsys_id IN ( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} )" }','To calculate the current monitor hit throughput, the metrics for each appliance are averaged over a 5-second interval. Additionally, the data ingestion latency is also set at 5 seconds.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-proxy-policy-rule-with-manipulation-action', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id AS rule_id, sub_action AS sub_action, SUM( hit_count ) AS hit_count, SUM( in_bytes + out_bytes ) AS bytes FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND action=48 AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id, sub_action ORDER BY ${metric} DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-by-rule-id', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, rule_id, RATE(hit_count, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_hit_count_per_sec, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN (${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')), rule_id ORDER BY stat_time ASC LIMIT ${limit}" }','The hit count and bytes of the favorite proxy policy rule over the selected time frame.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-ssl-intercept-pinning-trend', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, AVG( CASE WHEN pinning_status = ''0'' THEN hit_count_per_sec ELSE 0 END ) AS avg_not_pinning_hit_count_per_sec, AVG( CASE WHEN pinning_status = ''1'' THEN hit_count_per_sec ELSE 0 END ) AS avg_pinning_hit_count_per_sec, AVG( CASE WHEN pinning_status = ''2'' THEN hit_count_per_sec ELSE 0 END ) AS avg_maybe_pinning_hit_count_per_sec FROM (SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')) AS stat_time, pinning_status, RATE( hit_count, SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS hit_count_per_sec FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND action = 2 AND ( ${filter} ) GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}'')), pinning_status ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }','Monitor SSL Pinning over the selected time frame.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-ssl-intercept-pinning-statistics', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT ( CASE WHEN pinning_status = ''0'' THEN ''not_pinning_num'' WHEN pinning_status = ''1'' THEN ''pinning_num'' WHEN pinning_status = ''2'' THEN ''maybe_pinning_num'' ELSE concat( pinning_status ) END ) AS type, SUM( hit_count ) AS hits FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND action = 2 AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY pinning_status ORDER BY pinning_status" }','The statistics of Proxy SSL Intercept pinning.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-by-action', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, (CASE WHEN action = 2 THEN ''Intercept'' WHEN action = 3 THEN ''No Intercept'' WHEN action = 48 THEN ''Manipulation'' ELSE concat(action) END ) AS action, RATE(hit_count, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_hit_count_per_sec, SUM(hit_count) AS total_hit_count FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN (${vsys_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')), action ORDER BY stat_time ASC LIMIT ${limit}" }','The proxy Intercept policy hit the count of action over the selected time frame.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-manipulation-policy-rule-hits-by-action', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, sub_action AS action, RATE(hit_count, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_hit_count_per_sec, SUM(hit_count) AS total_hit_count FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN (${vsys_id}) AND action = 48 AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')), sub_action ORDER BY stat_time ASC LIMIT ${limit}" }','The proxy manipulation policy hit the count of action over the selected time frame.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-proxy-policy-rule', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, SUM( hit_count ) AS hit_count, SUM( in_bytes + out_bytes ) AS bytes FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id ORDER BY ${metric} DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-statistics', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, SUM( hit_count ) AS hits, SUM( in_bytes + out_bytes ) AS bytes FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id LIMIT ${limit}" }','The statistics of specific rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-trend', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) AS stat_time, SUM( hit_count ) AS hits FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'' )) LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-first-and-last-hit', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, DATE_FORMAT( MIN( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS first_hit, DATE_FORMAT( MAX( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS last_hit FROM proxy_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The time of the first and last hit for policy rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-policy-rule-hits-current-throughput', 'proxy_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT SUM( in_bytes )* 8 / 5 AS in_bps, SUM( out_bytes )* 8 / 5 AS out_bps, SUM( in_bytes + out_bytes )* 8 / 5 AS total_bps FROM proxy_rule_hits WHERE __time >= FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 15 ) AND __time < FROM_UNIXTIME( UNIX_TIMESTAMP( now())- 10 ) AND vsys_id IN ( ${vsys_id} ) AND rule_id IN ( ${rule_id} ) AND ( ${filter} )" }','To calculate the current policy hit throughput, the metrics for each appliance are averaged over a 15-second interval. Additionally, the data ingestion latency is also set at 15 seconds.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-hits-first-and-last-hit', 'statistics_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, DATE_FORMAT( min( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS first_hit, DATE_FORMAT( max( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS last_hit FROM statistics_rule_hits WHERE __time >= ''${start_time}'' AND __time <''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The time of the first and last hit for statistics policy rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-statistics', 'statistics_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, sum( hit_count ) AS hits, sum( in_bytes + out_bytes ) AS bytes FROM statistics_rule_hits WHERE __time >= ''${start_time}'' AND __time <''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The statistics of specific rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-hits-trend', 'statistics_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, RATE(in_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_in_bits_per_sec, RATE(out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_out_bits_per_sec, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec, SUM(in_bytes) AS total_in_bytes, SUM(out_bytes) AS total_out_bytes, SUM( in_bytes + out_bytes ) AS total_bytes, RATE(hit_count, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_hit_count_per_sec, SUM(hit_count) AS total_hit_count FROM statistics_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN (${vsys_id}) AND rule_id IN (${rule_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-protection-rule-hits-first-and-last-hit', 'dos_protection_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, DATE_FORMAT( min( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS first_hit, DATE_FORMAT( max( __time ), ''%Y-%m-%d %H:%i:%s'' ) AS last_hit FROM dos_protection_rule_hits WHERE __time >= ''${start_time}'' AND __time <''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The time of the first and last hit for the dos protection policy rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-protection-rule-statistics', 'dos_protection_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT rule_id, sum( hit_count ) AS hits, sum( in_bytes + out_bytes ) AS bytes FROM dos_protection_rule_hits WHERE __time >= ''${start_time}'' AND __time <''${end_time}'' AND rule_id IN ( ${rule_id} ) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY rule_id" }','The statistics of specific rules.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-protection-rule-hits-trend', 'dos_protection_rule_hits', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, RATE(in_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_in_bits_per_sec, RATE(out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_out_bits_per_sec, RATE(in_bytes + out_bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 AS avg_bits_per_sec, SUM(in_bytes) AS total_in_bytes, SUM(out_bytes) AS total_out_bytes, SUM( in_bytes + out_bytes ) AS total_bytes, RATE(hit_count, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_hit_count_per_sec, SUM(hit_count) AS total_hit_count FROM dos_protection_rule_hits WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN (${vsys_id}) AND rule_id IN (${rule_id}) AND (${filter}) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-line-chart-template', 'statistics_rule', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time [[,RATE(${metric_in_bytes}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1)*8 AS avg_in_bits_per_sec]] [[,RATE(${metric_out_bytes}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1)*8 AS avg_out_bits_per_sec]] [[,RATE(${metric_bytes}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1)*8 AS avg_bits_per_sec]] [[,RATE(${metric_in_pkts}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_in_pkts_per_sec]] [[,RATE(${metric_out_pkts}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_out_pkts_per_sec]] [[,RATE(${metric_pkts}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_pkts_per_sec]] [[,RATE(${metric_new_c2s_flows}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_new_c2s_flows_per_sec]] [[,RATE(${metric_new_s2c_flows}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_new_s2c_flows_per_sec]] [[,RATE(${metric_new_in_sessions}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_new_in_sessions_per_sec]] [[,RATE(${metric_new_out_sessions}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_new_out_sessions_per_sec]] [[,RATE(${metric_sessions}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_sessions_per_sec]] [[,RATE(${metric_new_unestablished_sessions}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_new_unestablished_sessions_per_sec]] [[,RATE(${metric_syn_pkts}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) AS avg_syn_pkts_per_sec]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_session_identifier_sketch}) AS active_sessions]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_client_ip_sketch}) AS unique_client_ips]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_server_ip_sketch}) AS unique_server_ips]] [[,QUANTILE_HDR(${metric_in_latency_ms_sketch},0.5) AS median_tcp_in_latency_ms]] [[,QUANTILE_HDR(${metric_in_latency_ms_sketch},0.95) AS p95th_tcp_in_latency_ms]] [[,QUANTILE_HDR(${metric_in_latency_ms_sketch},0.99) AS p99th_tcp_in_latency_ms]] [[,QUANTILE_HDR(${metric_out_latency_ms_sketch},0.5) AS median_tcp_out_latency_ms]] [[,QUANTILE_HDR(${metric_out_latency_ms_sketch},0.95) AS p95th_tcp_out_latency_ms]] [[,QUANTILE_HDR(${metric_out_latency_ms_sketch},0.99) AS p99th_tcp_out_latency_ms]] [[,QUANTILE_HDR(${metric_latency_ms_sketch},0.5) AS median_tcp_latency_ms]] [[,QUANTILE_HDR(${metric_latency_ms_sketch},0.95) AS p95th_tcp_latency_ms]] [[,QUANTILE_HDR(${metric_latency_ms_sketch},0.99) AS p99th_tcp_latency_ms]] [[,QUANTILE_HDR(${metric_in_pkt_length_sketch},0.5) AS median_in_pkt_length]] [[,QUANTILE_HDR(${metric_in_pkt_length_sketch},0.95) AS p95th_in_pkt_length]] [[,QUANTILE_HDR(${metric_in_pkt_length_sketch},0.99) AS p99th_in_pkt_length]] [[,QUANTILE_HDR(${metric_out_pkt_length_sketch},0.5) AS median_out_pkt_length]] [[,QUANTILE_HDR(${metric_out_pkt_length_sketch},0.95) AS p95th_out_pkt_length]] [[,QUANTILE_HDR(${metric_out_pkt_length_sketch},0.99) AS p99th_out_pkt_length]] [[,QUANTILE_HDR(${metric_pkt_length_sketch},0.5) AS median_pkt_length]] [[,QUANTILE_HDR(${metric_pkt_length_sketch},0.95) AS p95th_pkt_length]] [[,QUANTILE_HDR(${metric_pkt_length_sketch},0.99) AS p99th_pkt_length]] FROM statistics_rule WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND rule_id = ${rule_id} AND template_id = ${template_id} AND chart_id = ${chart_id} AND version = ${version} AND ( ${filter} ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-histogram-chart-template', 'statistics_rule', 'qgw', 'sql', '{ "statement": "SELECT [[PERCENTILES_HDR(${metric_in_latency_ms_sketch}) AS tcp_in_latency, QUANTILE_HDR(${metric_in_latency_ms_sketch}, 0.5) as tcp_in_latency_quantiles]] [[,PERCENTILES_HDR(${metric_out_latency_ms_sketch}) AS tcp_out_latency ,QUANTILE_HDR(${metric_out_latency_ms_sketch}, 0.5) as tcp_out_latency_quantiles]] [[,PERCENTILES_HDR(${metric_latency_ms_sketch}) AS tcp_latency ,QUANTILE_HDR(${metric_latency_ms_sketch}, 0.5) as tcp_latency_quantiles]] FROM statistics_rule WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( ${vsys_id} ) AND rule_id = ${rule_id} AND template_id = ${template_id} AND chart_id = ${chart_id} AND version = ${version} AND ( ${filter} ) LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('statistics-rule-table-or-bar-chart-template', 'statistics_rule', 'qgw', 'sql', '{ "statement": "SELECT [[${dimension_device_group} AS device_group]] [[,${dimension_client_ip} AS client_ip]] [[,${dimension_server_ip} AS server_ip]] [[,${dimension_client_asn} AS client_asn]] [[,${dimension_server_asn} AS server_asn]] [[,${dimension_client_country} AS client_country]] [[,${dimension_server_country} AS server_country]] [[,${dimension_server_fqdn} AS server_fqdn]] [[,${dimension_server_domain} AS server_domain]] [[,${dimension_application} AS application]] [[,${dimension_fqdn_category} AS fqdn_category]] [[,${dimension_client_ip_object} AS client_ip_object]] [[,${dimension_server_ip_object} AS server_ip_object]] [[,${dimension_c2s_ttl} AS c2s_ttl]] [[,${dimension_s2c_ttl} AS s2c_ttl]] [[,${dimension_c2s_link_id} AS c2s_link_id]] [[,${dimension_s2c_link_id} AS s2c_link_id]] [[,${dimension_client_port} AS client_port]] [[,${dimension_server_port} AS server_port]] [[,sum(${metric_in_bytes}) AS in_bytes]] [[,sum(${metric_out_bytes}) AS out_bytes]] [[,sum(${metric_bytes}) AS bytes]] [[,sum(${metric_in_pkts}) AS in_pkts]] [[,sum(${metric_out_pkts}) AS out_pkts]] [[,sum(${metric_pkts}) AS pkts]] [[,sum(${metric_new_c2s_flows}) AS new_c2s_flows]] [[,sum(${metric_new_s2c_flows}) AS new_s2c_flows]] [[,sum(${metric_new_in_sessions}) AS new_in_sessions]] [[,sum(${metric_new_out_sessions}) AS new_out_sessions]] [[,sum(${metric_sessions}) AS sessions]] [[,sum(${metric_new_unestablished_sessions}) AS new_unestablished_sessions]] [[,sum(${metric_syn_pkts}) AS syn_pkts]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_session_identifier_sketch}) AS active_sessions]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_client_ip_sketch}) AS unique_client_ips]] [[,APPROX_COUNT_DISTINCT_HLLD(${metric_server_ip_sketch}) AS unique_server_ips]] FROM statistics_rule WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id in (${vsys_id}) AND rule_id = ${rule_id} AND template_id = ${template_id} AND chart_id = ${chart_id} AND version = ${version} AND ( ${filter} ) GROUP BY [[${dimension_device_group}]] [[,${dimension_client_ip}]] [[,${dimension_server_ip}]] [[,${dimension_client_asn}]] [[,${dimension_server_asn}]] [[,${dimension_client_country}]] [[,${dimension_server_country}]] [[,${dimension_server_fqdn}]] [[,${dimension_server_domain}]] [[,${dimension_application}]] [[,${dimension_fqdn_category}]] [[,${dimension_client_ip_object}]] [[,${dimension_server_ip_object}]] [[,${dimension_c2s_ttl}]] [[,${dimension_s2c_ttl}]] [[,${dimension_c2s_link_id}]] [[,${dimension_s2c_link_id}]] [[,${dimension_client_port}]] [[,${dimension_server_port}]] ORDER BY ${order_by} LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-client-ip', 'top_client_ips', 'qgw', 'sql', '{ "statement": "SELECT client_ip AS client_ip, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( pkts ) AS packets FROM top_client_ips WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND rank_by=''${metric}'' AND ( ${filter} ) GROUP BY client_ip ORDER BY ${metric} DESC LIMIT ${limit}" }','Aggregate traffic sessions, bytes, and packets based on the client''s IP address. The ${metric} variable includes three options: sessions, bytes, and packets.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-server-ip', 'top_server_ips', 'qgw', 'sql', '{ "statement": "SELECT server_ip AS server_ip, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( pkts ) AS packets FROM top_server_ips WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND rank_by=''${metric}'' AND ( ${filter} ) GROUP BY server_ip ORDER BY ${metric} DESC LIMIT ${limit}" }','Aggregate traffic sessions, bytes, and packets based on the server IP address. The ${metric} variable includes three options: sessions, bytes, and packets.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-internal-ip', 'top_internal_ips', 'qgw', 'sql', '{ "statement": "SELECT internal_ip AS internal_ip, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( pkts ) AS packets FROM top_internal_ips WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND rank_by=''${metric}'' AND ( ${filter} ) GROUP BY internal_ip ORDER BY ${metric} DESC LIMIT ${limit}" }','Aggregate traffic sessions, bytes, and packets based on the internal IP address. The ${metric} variable includes three options: sessions, bytes, and packets.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-external-ip', 'top_external_ips', 'qgw', 'sql', '{ "statement": "SELECT external_ip AS external_ip, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( pkts ) AS packets FROM top_external_ips WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND rank_by=''${metric}'' AND ( ${filter} ) GROUP BY external_ip ORDER BY ${metric} DESC LIMIT ${limit}" }','Aggregate traffic sessions, bytes, and packets based on the external IP address. The ${metric} variable includes three options: sessions, bytes, and packets.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-server-domain', 'top_server_domains', 'qgw', 'sql', '{ "statement": "SELECT server_domain, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( pkts ) AS packets FROM top_server_domains WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND rank_by=''${metric}'' AND ( ${filter} ) GROUP BY server_domain ORDER BY ${metric} DESC LIMIT ${limit}" }','Aggregate traffic sessions, bytes, and packets based on the server domains. The ${metric} variable includes three options: sessions, bytes, and packets.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-server-fqdn', 'top_server_fqdns', 'qgw', 'sql', '{ "statement": "SELECT server_fqdn, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( pkts ) AS packets FROM top_server_fqdns WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND rank_by=''${metric}'' AND ( ${filter} ) GROUP BY server_fqdn ORDER BY ${metric} DESC LIMIT ${limit}" }','Aggregate traffic sessions, bytes, and packets based on the server FQDNs. The ${metric} variable includes three options: sessions, bytes, and packets.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-client-country', 'top_client_countries', 'qgw', 'sql', '{ "statement": "SELECT client_country AS client_country, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( pkts ) AS packets FROM top_client_countries WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND rank_by=''${metric}'' AND ( ${filter} ) GROUP BY client_country ORDER BY ${metric} DESC LIMIT ${limit}" }','Aggregate traffic sessions, bytes, and packets based on the client''s country or region. The ${metric} variable includes three options: sessions, bytes, and packets.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-server-country', 'top_server_countries', 'qgw', 'sql', '{ "statement": "SELECT server_country AS server_country, SUM( sessions) AS sessions, SUM( out_bytes ) AS out_bytes, SUM( in_bytes ) AS in_bytes, SUM( bytes ) AS bytes, SUM( out_pkts ) AS out_packets, SUM( in_pkts ) AS in_packets, SUM( pkts ) AS packets FROM top_server_countries WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND rank_by=''${metric}'' AND ( ${filter} ) GROUP BY server_country ORDER BY ${metric} DESC LIMIT ${limit}" }','Aggregate traffic sessions, bytes, and packets based on the server''s country or region. The ${metric} variable includes three options: sessions, bytes, and packets.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-application', 'application_protocol_stat', 'qgw', 'sql', '{ "statement": "SELECT app_name, SUM( sessions) AS sessions, SUM( in_bytes ) AS in_bytes, SUM( out_bytes ) AS out_bytes, SUM( in_bytes + out_bytes ) AS bytes, SUM( in_pkts ) AS in_packets, SUM( out_pkts ) AS out_packets, SUM( in_pkts + out_pkts ) AS packets FROM application_protocol_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN( ${vsys_id} ) AND app_name IS NOT NULL AND ( ${filter} ) GROUP BY app_name ORDER BY ${metric} DESC LIMIT ${limit}" }','Aggregate traffic sessions, bytes, and packets based on the applications. The ${metric} variable includes three options: sessions, bytes, and packets.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('session-record-list', 'session_record', 'qgw', 'sql', '{ "statement": "SELECT ${columns} FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('session-record-count', 'session_record', 'qgw', 'sql', '{ "statement": "SELECT count(1) as count FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND ( ${filter})" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('session-record-timeline', 'session_record', 'qgw', 'sql', '{ "statement": "select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, decoded_as as type, COUNT(1) as sessions, SUM(sent_bytes + received_bytes) as bytes, SUM(sent_pkts + received_pkts) as packets FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) GROUP BY stat_time, decoded_as" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('transaction-record-list', 'transaction_record', 'qgw', 'sql', '{ "statement": "SELECT ${columns} FROM transaction_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('transaction-record-count', 'transaction_record', 'qgw', 'sql', '{ "statement": "SELECT count(1) as count FROM transaction_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND ( ${filter})" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('security-event-list', 'security_event', 'qgw', 'sql', '{ "statement": "SELECT ${columns} FROM security_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('security-event-count', 'security_event', 'qgw', 'sql', '{ "statement": "SELECT count(1) as count FROM security_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND ( ${filter})" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('security-event-timeline', 'security_event', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, security_action as type, COUNT(1) as sessions, SUM(sent_bytes + received_bytes) as bytes, SUM(sent_pkts + received_pkts) as packets FROM security_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND notEmpty(security_action) AND ( ${filter}) group by stat_time, security_action order by stat_time asc" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('monitor-event-list', 'monitor_event', 'qgw', 'sql', '{ "statement": "SELECT ${columns} FROM monitor_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND (${filter}) ORDER BY recv_time DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('monitor-event-count', 'monitor_event', 'qgw', 'sql', '{ "statement": "SELECT count(1) as count FROM monitor_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND (${filter})" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('monitor-event-timeline', 'monitor_event', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time,CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'')) AS stat_time, COUNT(1) as sessions FROM monitor_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND (${filter}) GROUP BY stat_time" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-event-intercept-list', 'session_record', 'qgw', 'sql', '{ "statement": "SELECT ${columns} FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND notEmpty(proxy_action) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-event-intercept-count', 'session_record', 'qgw', 'sql', '{ "statement": "SELECT count(1) as count FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND notEmpty(proxy_action) AND ( ${filter})" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-event-intercept-timeline', 'session_record', 'qgw', 'sql', '{ "statement": "select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, decoded_as as type, COUNT(1) as sessions, SUM(sent_bytes + received_bytes) as bytes, SUM(sent_pkts + received_pkts) as packets FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND notEmpty(proxy_action) AND ( ${filter}) GROUP BY stat_time, decoded_as" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-event-manipulation-list', 'proxy_event', 'qgw', 'sql', '{ "statement": "SELECT ${columns} FROM proxy_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id in (${vsys_id}) AND ( ${filter} ) ORDER BY recv_time DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-event-manipulation-count', 'proxy_event', 'qgw', 'sql', '{ "statement": "SELECT COUNT(1) as count FROM proxy_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id in (${vsys_id}) AND ( ${filter} )" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('proxy-event-manipulation-timeline', 'proxy_event', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, proxy_action as type, COUNT(1) as events FROM proxy_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND notEmpty(proxy_action) AND ( ${filter} ) GROUP BY stat_time, proxy_action ORDER BY stat_time asc" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('voip-record-list', 'voip_record', 'qgw', 'sql', '{ "statement": "SELECT ${columns} FROM voip_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id in(${vsys_id}) AND ( ${filter}) ORDER BY recv_time DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('voip-record-count', 'voip_record', 'qgw', 'sql', '{ "statement": "SELECT count(1) as count FROM voip_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id in(${vsys_id}) AND ( ${filter})" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('voip-record-timeline', 'voip_record', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, decoded_as as type, COUNT(1) as count FROM voip_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND ( ${filter}) GROUP BY stat_time,decoded_as ORDER BY stat_time asc" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-top-source-countries', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT arrayJoin(splitByString('','',source_country_list)) AS source_country, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND notEmpty(source_country_list) AND vsys_id IN ( ${vsys_id} ) AND ( ${filter} ) GROUP BY arrayJoin(splitByString('','',source_country_list)) ORDER BY count DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-top-destination-countries', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT destination_country, COUNT(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY destination_country ORDER BY count DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-top-victims', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT destination_ip, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY destination_ip ORDER BY count DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-attack-type', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT attack_type, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY attack_type ORDER BY attack_type LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-severity', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT severity, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY severity ORDER BY severity LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-destination-ip-distribution', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT destination_ip, any(destination_country) AS destination_country, groupUniqArray(arrayJoin(splitByString('','', source_country_list))) AS source_coutries, MAX(bit_rate) AS max_bit_rate, MAX(packet_rate) AS max_packet_rate, MAX(session_rate) AS max_session_rate, FROM_UNIXTIME(MIN(start_time)) AS first_active_time, FROM_UNIXTIME(MAX(end_time)) AS last_active_time, MAX_DURATION(end_time, 600) AS max_duration, groupUniqArray(attack_type) AS attack_type, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY destination_ip ORDER BY count DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-attack-connection', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(recv_time) AS stat_time, destination_country, source_country_list, attack_type, severity, bit_rate, bytes, packet_rate, packets, session_rate, sessions FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND (${filter}) ORDER BY recv_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-attack-volume-summary', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT RATE(bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 as avg_bits_per_sec FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND (${filter})" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-threat-map-attack-volume-trend', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, RATE(bytes, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), 1) * 8 as avg_bits_per_sec FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY stat_time ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-event-timeline', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'')) AS stat_time, attack_type AS type, count(*) AS count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) GROUP BY stat_time, attack_type ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-event-count', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT count(1) as count FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} )" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dos-event-list', 'dos_event', 'qgw', 'sql', '{ "statement": "SELECT ${columns} FROM dos_event WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN (${vsys_id}) AND ( ${filter} ) ORDER BY recv_time DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('datapath-telemetry-record-count', 'datapath_telemetry_record', 'qgw', 'sql', '{ "statement": "SELECT count(1) as count FROM datapath_telemetry_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id in(${vsys_id}) AND ( ${filter})" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('datapath-telemetry-record-list', 'datapath_telemetry_record', 'qgw', 'sql', '{ "statement": "SELECT ${columns} FROM datapath_telemetry_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND vsys_id IN(${vsys_id}) AND (${filter}) ORDER BY timestamp_us ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('datapath-telemetry-packet-combine', 'datapath_telemetry_record', 'qgw', 'dsl', '{"id":"${job_id}","name":"datapath_telemetry_packet_combine","data_source":"datapath_telemetry_record","filter":"job_id=''${job_id}'' AND vsys_id in (${vsys_id}) AND (${filter})"}',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-discovery', 'statistics', 'qgw', 'dsl', '{ "name": "field_discovery", "data_source": "${source}", "filter": "vsys_id in (${vsys_id}) AND (${filter})", "custom.field_discovery.metric": "${metric}", "custom.field_discovery.metric.fn": "${fn}", "custom.field_discovery.fields": ["${field_list}"] }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-top-values', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT ${column_name}, count(*) as cnt FROM ${source} where recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY ${column_name} ORDER BY cnt DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-rare-values', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT ${column_name}, count(*) as cnt FROM ${source} where recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY ${column_name} ORDER BY cnt ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-avg-value-over-time', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT TIME_FLOOR_WITH_FILL(${unix_timestamp_column}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'') AS stat_time, avg(${column_name_long}) FROM ${source} WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY TIME_FLOOR_WITH_FILL(${unix_timestamp_column}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'') ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-max-value-over-time', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT TIME_FLOOR_WITH_FILL(${unix_timestamp_column}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'') AS stat_time, max(${column_name_long}) FROM ${source} WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY TIME_FLOOR_WITH_FILL(${unix_timestamp_column}, CHART_GRANULARITY(''${start_time}'', ''${end_time}''), ''zero'') ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-min-value-over-time', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT TIME_FLOOR_WITH_FILL(${unix_timestamp_column},CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'') AS stat_time, min(${column_name_long}) FROM ${source} WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY TIME_FLOOR_WITH_FILL(${unix_timestamp_column},CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'') ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-median-value-over-time', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT TIME_FLOOR_WITH_FILL(${unix_timestamp_column},CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'') AS stat_time, MEDIAN(${column_name_long}) FROM ${source} WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY TIME_FLOOR_WITH_FILL(${unix_timestamp_column},CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'') ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-p95-value-over-time', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT TIME_FLOOR_WITH_FILL(${unix_timestamp_column},CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'') AS stat_time, QUANTILE(${column_name_long},0.95) FROM ${source} WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY TIME_FLOOR_WITH_FILL(${unix_timestamp_column},CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'') ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-p99-value-over-time', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT TIME_FLOOR_WITH_FILL(${unix_timestamp_column},CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'') AS stat_time, QUANTILE(${column_name_long},0.99) FROM ${source} WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY TIME_FLOOR_WITH_FILL(${unix_timestamp_column},CHART_GRANULARITY(''${start_time}'', ''${end_time}''),''zero'') ORDER BY stat_time ASC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-top-frequent-elements-in-array', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT item as ${column_name_array}, sum(cnt) as cnt FROM( SELECT arrayJoin(items) as item, cnt FROM ( SELECT IF(empty(${column_name_array}), arrayPushBack(${column_name_array}, NULL), ${column_name_array}) as items, count(*) as cnt FROM ${source} as source where recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY ${column_name_array})) GROUP BY item ORDER BY cnt DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-top-frequent-elements-in-string-of-array', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT item as ${column_name_string}, sum(cnt) as cnt FROM( SELECT arrayJoin(items) as item, cnt FROM ( SELECT splitByString(''${separator}'',${column_name_string}) as items, count(*) as cnt FROM ${source} as source where recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} AND notEmpty(${column_name_string}) GROUP BY ${column_name_string})) GROUP BY item ORDER BY cnt DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('field-statistics-top-frequent-elements-in-long-of-bit', 'statistics', 'qgw', 'sql', '{ "statement":"SELECT item as ${column_name_long}, sum(cnt) as cnt FROM( SELECT arrayJoin(items) as item, cnt FROM ( SELECT bitmaskToArray(${column_name_long}) as items, count(*) as cnt FROM ${source} as source where recv_time >= UNIX_TIMESTAMP(''${start_time}'') AND recv_time < UNIX_TIMESTAMP(''${end_time}'') AND ${filter} GROUP BY ${column_name_long})) GROUP BY item ORDER BY cnt DESC LIMIT ${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('ip-learning-fqdn-relate-ip', 'ip_learning', 'qgw', 'dsl', '{ "name": "ip-learning-fqdn-relate-ip", "data_source":"ip_learnging_view", "filter": "vsys_id in(${vsys_id}) AND (${filter})", "intervals": ["${start_time}/${end_time}"], "limit": "${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('ip-learning-active-ip', 'ip_learning', 'qgw', 'dsl', '{ "name": "ip-learning-active-ip", "data_source":"active_ip_view", "filter": "vsys_id in(${vsys_id}) AND (${filter})", "intervals": ["${start_time}/${end_time}"], "order_by": "${order_by}", "limit": "${limit}" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-frequent-elements-in-long-of-bit', 'benchmark', 'qgw', 'sql', '{ "statement": "select item,sum(count) as count from( select arrayJoin(items) as item, count from ( select bitmaskToArray(flags) as items,count(*) as count from session_record as sr where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') group by flags)) group by item order by count desc", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-frequent-elements-in-flags-with-label', 'benchmark', 'qgw', 'sql', '{ "statement": "select flag, sum(sessions) as sessions from( select arrayJoin(array( if(bitAnd(flags, 1)= 1, ''Asymmetric'', ''''), if(bitAnd(flags, 2)= 2, ''Bulky'', ''''), if(bitAnd(flags, 4)= 4, ''CBR Streaming'', ''''), if(bitAnd(flags, 8)= 8, ''Client is Local'', ''''), if(bitAnd(flags, 16)= 16, ''Server is Local'', ''''), if(bitAnd(flags, 32)= 32, ''Download'', ''''), if(bitAnd(flags, 64)= 64, ''Interactive'', ''''), if(bitAnd(flags, 128)= 128, ''Inbound'', ''''), if(bitAnd(flags, 256)= 256, ''Outbound'', ''''), if(bitAnd(flags, 512)= 512, ''Pseudo Unidirectional'', ''''), if(bitAnd(flags, 1024)= 1024, ''Streaming'', ''''), if(bitAnd(flags, 2048)= 2048, ''Unidirectional'', ''''), if(bitAnd(flags, 4096)= 4096, ''Random looking'', ''''), if(bitAnd(flags, 8192)= 8192, ''C2S'', ''''), if(bitAnd(flags, 16384)= 16384, ''S2C'', ''''), if(bitAnd(flags, 32768)= 32768, ''Bidirectional'', ''''), if(flags=0, ''N/A'', ''''))) as flag , bytes, packets, sessions from ( select flags, count(*) as sessions, sum(sent_bytes + received_bytes) as bytes, sum(sent_pkts + received_pkts) as packets from session_record as sr where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') group by flags ) ) where notEmpty(flag) group by flag order by sessions desc", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-frequent-elements-in-string-of-array', 'benchmark', 'qgw', 'sql', '{ "statement": "select item, sum(count) as count from( select arrayJoin(items) as item, count from ( select splitByString(''.'',decoded_path) as items, count(*) as count from session_record as sr where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and notEmpty(decoded_path) group by decoded_path)) group by item order by count desc limit 10", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-frequent-elements-in-array-use-sub-query', 'benchmark', 'qgw', 'sql', '{ "statement": "select item, sum(count) as count,sum(bytes) from( select arrayJoin(items) as item, count, bytes from ( select monitor_rule_list as items, count(*) as count,sum(sent_bytes+received_bytes) as bytes from session_record as sr where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and notEmpty(monitor_rule_list) group by monitor_rule_list)) group by item order by count desc limit 20", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-frequent-elements-in-array', 'benchmark', 'qgw', 'sql', '{ "statement": "select arrayJoin(monitor_rule_list) as item, count(*) as count,sum(sent_bytes+received_bytes) as bytes from session_record as sr where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and notEmpty(monitor_rule_list) group by arrayJoin(monitor_rule_list) order by count desc limit 20", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('nested-json-parser-by-app-id', 'benchmark', 'qgw', 'sql', '{ "statement": "select app_debug_info, JSONExtract(app_debug_info, ''Tuple(UNKNOWN Nested(app_name String, app_id UInt32),THIRD Nested(app_name String, app_id UInt32),USER_DEFINE Nested(app_name String, app_id UInt32))'') as parsed_json, tupleElement(tupleElement(parsed_json,''THIRD''),''app_name'') THIRD_app_name, tupleElement(tupleElement(parsed_json,''USER_DEFINE''),''app_name'') USER_DEFINE_app_name from session_record sr where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and notEmpty(app_debug_info) and has(THIRD_app_name,''ssl'') group by app_debug_info", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('log-summary', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(tcp_logs / logs,4) as \\"TCP Percentage\\", round(udp_logs / logs,4) as \\"UDP Percentage\\", round(out_bytes / bytes,4) as \\"Outgoing Percentage\\", round(in_bytes / bytes,4) as \\"Incoming Percentage\\", formatReadableQuantity(tcp_logs) as \\"TCP Logs\\", formatReadableQuantity(udp_logs) as \\"UDP Logs\\", formatReadableQuantity(logs) as \\"Logs\\", formatReadableSize(out_bytes) as \\"Outgoing Bytes\\", formatReadableSize(in_bytes) as \\"Incoming Bytes\\", formatReadableSize(bytes) as \\"Bytes\\" from( select sum(if(ip_protocol=''tcp'', 1, 0)) as tcp_logs, sum(if(ip_protocol=''udp'', 1, 0)) as udp_logs, sum(if(bitAnd(flags, 8) = 8, sent_bytes, received_bytes))as out_bytes, sum(if(bitAnd(flags, 8) = 8, received_bytes, sent_bytes)) as in_bytes, count(*) as logs, sum(sent_bytes + received_bytes) as bytes from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}''))" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('decoded-as', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(base_logs / total_logs, 4) as \\"BASE Percentage\\", round(http_logs / total_logs, 4) as \\"HTTP Percentage\\", round(ssl_logs / total_logs, 4) as \\"SSL Percentage\\", round(dns_logs / total_logs, 4) as \\"DNS Percentage\\", round(mail_logs / total_logs, 4) as \\"MAIL Percentage\\", round(rtp_logs / total_logs, 4) as \\"RTP Percentage\\", round(sip_logs / total_logs, 4) as \\"SIP Percentage\\", round(ftp_logs / total_logs, 4) as \\"FTP Percentage\\", base_logs as \\"BASE Logs\\", http_logs as \\"HTTP Logs\\", ssl_logs as \\"SSL Logs\\", dns_logs as \\"DNS Logs\\", mail_logs as \\"MAIL Logs\\", rtp_logs as \\"RTP Logs\\", sip_logs as \\"SIP Logs\\", ftp_logs as \\"FTP Logs\\", total_logs as \\"Total Logs\\" from( select sum(if(decoded_as=''BASE'', 1, 0)) as base_logs, sum(if(decoded_as=''HTTP'', 1, 0)) as http_logs, sum(if(decoded_as=''SSL'', 1, 0)) as ssl_logs, sum(if(decoded_as=''DNS'', 1, 0)) as dns_logs, sum(if(decoded_as=''MAIL'', 1, 0)) as mail_logs, sum(if(decoded_as=''RTP'', 1, 0)) as rtp_logs, sum(if(decoded_as=''SIP'', 1, 0)) as sip_logs, sum(if(decoded_as=''FTP'', 1, 0)) as ftp_logs, count(*) as total_logs from session_record as sub_connection where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}''))" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('endpoints', 'benchmark', 'qgw', 'sql', '{ "statement" : "select uniq(client_ip) as \\"Client IPs\\", uniq(server_ip) as \\"Server IPs\\", uniq(server_domain) as \\"Domains\\", uniq(http_host) as \\"Hosts\\", uniq(ssl_sni) as \\"SNIs\\" , uniq(client_ip, server_ip) as \\"Client and Server IPs\\" from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'')" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('endpoints-of-tcp', 'benchmark', 'qgw', 'sql', '{ "statement" : "select uniq(client_ip) as \\"Client IPs\\", uniq(server_ip) as \\"Server IPs\\", uniq(server_domain) as \\"Domains\\", uniq(http_host) as \\"Hosts\\", uniq(ssl_sni) as \\"SNIs\\" , uniq(client_ip, server_ip) as \\"Client and Server IPs\\" from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and ip_protocol =''tcp''" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('endpoints-of-udp', 'benchmark', 'qgw', 'sql', '{ "statement" : "select uniq(client_ip) as \\"Client IPs\\", uniq(server_ip) as \\"Server IPs\\", uniq(server_domain) as \\"Domains\\", uniq(http_host) as \\"Hosts\\", uniq(ssl_sni) as \\"SNIs\\" , uniq(client_ip, server_ip) as \\"Client and Server IPs\\" from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and ip_protocol =''udp''" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('tcp-percentile-distribution-bytes', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(QUANTILE(bytes,0.9999),2) as p9999, round(QUANTILE(bytes,0.999),2) as p999, round(QUANTILE(bytes,0.996),2) as p996, round(QUANTILE(bytes,0.995),2) as p995, round(QUANTILE(bytes,0.99),2) as p99, round(QUANTILE(bytes,0.98),2) as p98, round(QUANTILE(bytes,0.96),2) as p96, round(QUANTILE(bytes,0.95),2) as p95, round(QUANTILE(bytes,0.92),2) as p92, round(QUANTILE(bytes,0.90),2) as p90, round(QUANTILE(bytes,0.89),2) as p89, round(QUANTILE(bytes,0.88),2) as p88, round(median(bytes),2) as p50 from( select server_ip, sum(sent_bytes+received_bytes) as bytes from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp'')) group by server_ip)" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('tcp-percentile-distribution-sessions', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(QUANTILE(sessions,0.9999),2) as p9999, round(QUANTILE(sessions,0.999),2) as p999, round(QUANTILE(sessions,0.996),2) as p996, round(QUANTILE(sessions,0.995),2) as p995, round(QUANTILE(sessions,0.99),2) as p99, round(QUANTILE(sessions,0.98),2) as p98, round(QUANTILE(sessions,0.96),2) as p96, round(QUANTILE(sessions,0.95),2) as p95, round(QUANTILE(sessions,0.92),2) as p92, round(QUANTILE(sessions,0.90),2) as p90, round(QUANTILE(sessions,0.89),2) as p89, round(QUANTILE(sessions,0.88),2) as p88, round(median(sessions),2) as p50 from( select server_ip, count(1) as sessions from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp'')) group by server_ip)" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('udp(53,443)-percentile-distribution-bytes', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(QUANTILE(bytes,0.9999),2) as p9999, round(QUANTILE(bytes,0.999),2) as p999, round(QUANTILE(bytes,0.996),2) as p996, round(QUANTILE(bytes,0.995),2) as p995, round(QUANTILE(bytes,0.99),2) as p99, round(QUANTILE(bytes,0.98),2) as p98, round(QUANTILE(bytes,0.96),2) as p96, round(QUANTILE(bytes,0.95),2) as p95, round(QUANTILE(bytes,0.92),2) as p92, round(QUANTILE(bytes,0.90),2) as p90, round(QUANTILE(bytes,0.89),2) as p89, round(QUANTILE(bytes,0.88),2) as p88, round(median(bytes),2) as p50 from( select server_ip, sum(sent_bytes+received_bytes) as bytes from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and ip_protocol in (''udp'') and server_port in (53,443) group by server_ip)" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('udp(53,443)-percentile-distribution-sessions', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(QUANTILE(sessions,0.9999),2) as p9999, round(QUANTILE(sessions,0.999),2) as p999, round(QUANTILE(sessions,0.996),2) as p996, round(QUANTILE(sessions,0.995),2) as p995, round(QUANTILE(sessions,0.99),2) as p99, round(QUANTILE(sessions,0.98),2) as p98, round(QUANTILE(sessions,0.96),2) as p96, round(QUANTILE(sessions,0.95),2) as p95, round(QUANTILE(sessions,0.92),2) as p92, round(QUANTILE(sessions,0.90),2) as p90, round(QUANTILE(sessions,0.89),2) as p89, round(QUANTILE(sessions,0.88),2) as p88, round(median(sessions),2) as p50 from( select server_ip, count(*) as sessions from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and ip_protocol in (''udp'') and server_port in (53,443) group by server_ip)" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('tcp-percentile-distribution-of-server-ip-relate-uniq-cip', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(QUANTILE(client_ips,0.9999),2) as p9999, round(QUANTILE(client_ips,0.999),2) as p999, round(QUANTILE(client_ips,0.996),2) as p996, round(QUANTILE(client_ips,0.995),2) as p995, round(QUANTILE(client_ips,0.99),2) as p99, round(QUANTILE(client_ips,0.98),2) as p98, round(QUANTILE(client_ips,0.96),2) as p96, round(QUANTILE(client_ips,0.95),2) as p95, round(QUANTILE(client_ips,0.92),2) as p92, round(QUANTILE(client_ips,0.90),2) as p90, round(QUANTILE(client_ips,0.89),2) as p89, round(QUANTILE(client_ips,0.88),2) as p88, round(median(client_ips),2) as p50 from( select server_ip, uniq(client_ip) as client_ips from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp'')) group by server_ip)" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('udp-percentile-distribution-of-server-ip-relate-uniq-cip', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(QUANTILE(client_ips,0.9999),2) as p9999, round(QUANTILE(client_ips,0.999),2) as p999, round(QUANTILE(client_ips,0.996),2) as p996, round(QUANTILE(client_ips,0.995),2) as p995, round(QUANTILE(client_ips,0.99),2) as p99, round(QUANTILE(client_ips,0.98),2) as p98, round(QUANTILE(client_ips,0.96),2) as p96, round(QUANTILE(client_ips,0.95),2) as p95, round(QUANTILE(client_ips,0.92),2) as p92, round(QUANTILE(client_ips,0.90),2) as p90, round(QUANTILE(client_ips,0.89),2) as p89, round(QUANTILE(client_ips,0.88),2) as p88, round(median(client_ips),2) as p50 from( select server_ip, uniq(client_ip) as client_ips from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp'')) group by server_ip)" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('tcp-session-percent-of-top-100-server-ip', 'benchmark', 'qgw', 'sql', '{ "statement" : "select sum(sessions) sessions, count(*) as server_ips,( select count(*) from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp''))) as total_sessions, round(sessions / total_sessions, 6) as percent_sessions_to_total from ( SELECT server_ip, count(*) AS sessions FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp'')) GROUP BY server_ip order by sessions desc limit 100 )" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('udp-session-percent-of-top-100-server-ip', 'benchmark', 'qgw', 'sql', '{ "statement" : "select sum(sessions) sessions, count(*) as server_ips,( select count(*) from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp''))) as total_sessions, round(sessions / total_sessions, 6) as percent_sessions_to_total from ( SELECT server_ip, count(*) AS sessions FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp'')) GROUP BY server_ip order by sessions desc limit 100 )" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('tcp-byte-percent-of-top-100-server-ip', 'benchmark', 'qgw', 'sql', '{ "statement" : "select sum(bytes) as bytes, count(*) as server_ips,( select sum(sent_bytes+received_bytes) as bytes from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp''))) as total_bytes, round(bytes / total_bytes, 6) as percent_bytes_to_total from ( SELECT server_ip, sum(sent_bytes+received_bytes) as bytes FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp'')) GROUP BY server_ip order by bytes desc limit 100 )" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('udp-byte-percent-of-top-100-server-ip', 'benchmark', 'qgw', 'sql', '{ "statement" : "select sum(bytes) as bytes, count(*) as server_ips,( select sum(sent_bytes+received_bytes) as bytes from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp''))) as total_bytes, round(bytes / total_bytes, 6) as percent_bytes_to_total from ( SELECT server_ip, sum(sent_bytes+received_bytes) as bytes FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp'')) GROUP BY server_ip order by bytes desc limit 100 )" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('tcp-client-ip-and-traffic-percent-of-top-100-server-ip', 'benchmark', 'qgw', 'sql', '{ "statement" : "select uniq(client_ip) as client_ips, count(*) as sessions,( select uniq(client_ip) as total_client_ips from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp''))) as total_client_ips, round(client_ips / total_client_ips,6) as percent_client_ips_to_total, ( select count(*) as total_sessions from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp'')) ) as total_sessions, round(sessions / total_sessions,6) as percent_sessions_to_total, sum(sent_bytes+received_bytes) as bytes, ( select sum(sent_bytes+received_bytes) as total_bytes from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp'')) ) as total_bytes, round(bytes / total_bytes,6) as percent_bytes_to_total FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp'')) and server_ip in ( SELECT server_ip FROM session_record as cc WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''tcp'')) GROUP BY server_ip order by uniq(server_ip) desc limit 10 )" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('udp-client-ip-and-traffic-percent-of-top-100-Server-ip', 'benchmark', 'qgw', 'sql', '{ "statement" : "select uniq(client_ip) as client_ips, count(*) as sessions,( select uniq(client_ip) as total_client_ips from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp''))) as total_client_ips, round(client_ips / total_client_ips,6) as percent_client_ips_to_total, ( select count(*) as total_sessions from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp'')) ) as total_sessions, round(sessions / total_sessions,6) as percent_sessions_to_total, sum(sent_bytes+received_bytes) as bytes, ( select sum(sent_bytes+received_bytes) as total_bytes from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp'')) ) as total_bytes, round(bytes / total_bytes,6) as percent_bytes_to_total FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp'')) and server_ip in ( SELECT server_ip FROM session_record as cc WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') AND (ip_protocol IN (''udp'')) GROUP BY server_ip order by uniq(server_ip) desc limit 10 )" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('session-percent-of-top-100-sni', 'benchmark', 'qgw', 'sql', '{ "statement" : "select sum(sessions) sessions, count(*) as ssl_snis,( select count(*) from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and notEmpty(ssl_sni)) as total_sessions, round(sessions / total_sessions, 6) as percent_sessions_to_total from ( SELECT ssl_sni, count(*) AS sessions FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and notEmpty(ssl_sni) GROUP BY ssl_sni order by sessions desc limit 100 )" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('byte-percent-of-top-100-sni', 'benchmark', 'qgw', 'sql', '{ "statement" : "select sum(bytes) as bytes, count(*) as ssl_snis,( select sum(sent_bytes+received_bytes) as bytes from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and notEmpty(ssl_sni)) as total_bytes, round(bytes / total_bytes, 6) as percent_bytes_to_total from ( SELECT ssl_sni, sum(sent_bytes+received_bytes) as bytes FROM session_record WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and notEmpty(ssl_sni) GROUP BY ssl_sni order by bytes desc limit 100 )" , "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-10-client-ip-with-others', 'benchmark', 'qgw', 'sql', '{ "statement": "SELECT client_ip, sum(sent_bytes + received_bytes) as bytes from session_record sr1 where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') group by client_ip order by bytes desc limit 10 union all select null, sum(sent_bytes + received_bytes) as bytes from session_record sr2 where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and sr2.client_ip not in(select client_ip from session_record sr where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') group by client_ip order by sum(sent_bytes + received_bytes) desc limit 10)", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('app-bitrate-per-server-ip', 'benchmark', 'qgw', 'sql', '{ "statement" : "SELECT app, round(median(traffic_bytes) * 8 / 1000 / 1000 / 300,2) AS \\"Medain Mbits/s\\", round(avg(traffic_bytes) * 8 / 1000 / 1000 / 300,2) AS \\"AVG Mbits / s\\", round(QUANTILE(traffic_bytes, 0.95) * 8 / 1000 / 1000 / 300,2) as \\"P95 Mbits / s\\" FROM( SELECT app, toDateTime(intDiv(toUInt32(toDateTime(toDateTime(recv_time))),300) * 300) as stat_time, round(sum(sent_bytes + received_bytes)/ uniq(server_ip),2) as traffic_bytes FROM session_record as ss WHERE recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') GROUP BY stat_time, app) group by app order by \\"AVG Mbits / s\\" desc", "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('http-url-length-distribution', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(max(url_length),2) as max, round(QUANTILE(url_length,0.9999),2) as p9999, round(QUANTILE(url_length,0.99),2) as p99, round(QUANTILE(url_length,0.95),2) as p95, round(QUANTILE(url_length,0.90),2) as p90, round(median(url_length),2) as p50 from(select length(http_url) as url_length from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and decoded_as=''HTTP'')", "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('dns-rr-length-distribution', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(max(rr_length),2) as max, round(QUANTILE(rr_length,0.9999),2) as p9999, round(QUANTILE(rr_length,0.99),2) as p99, round(QUANTILE(rr_length,0.95),2) as p95, round(QUANTILE(rr_length,0.90),2) as p90, round(median(rr_length),2) as p50 from(select length(dns_rr) as rr_length from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and decoded_as=''DNS'')", "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('ssl-san-length-distribution', 'benchmark', 'qgw', 'sql', '{ "statement" : "select round(max(san_length),2) as max, round(QUANTILE(san_length,0.9999),2) as p9999, round(QUANTILE(san_length,0.99),2) as p99, round(QUANTILE(san_length,0.95),2) as p95, round(QUANTILE(san_length,0.90),2) as p90, round(median(san_length),2) as p50 from(select length(ssl_san) as san_length from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and decoded_as=''SSL'')", "output_mode":"json", "execution_mode":"oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('averages-usage-analysis', 'benchmark', 'qgw', 'sql', '{ "statement": "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL ( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}''))) AS stat_time, sum( in_bytes + out_bytes )* 8 / 1800 / 1000 / 1000 AS normal_rate_mbps, sum( in_bytes + out_bytes )/ count(DISTINCT (TIME_FLOOR_WITH_FILL ( UNIX_TIMESTAMP( __time ), SAMPLE_GRANULARITY(''${start_time}'', ''${end_time}''))))* 8 / 15 / 1000 / 1000 AS usage_rate_mbps FROM traffic_general_stat WHERE __time >= ''${start_time}'' AND __time < ''${end_time}'' AND vsys_id IN ( 1 ) GROUP BY TIME_FLOOR_WITH_FILL ( UNIX_TIMESTAMP( __time ), CHART_GRANULARITY(''${start_time}'', ''${end_time}'')) ORDER BY stat_time ASC", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('top-100-slowest-domains', 'benchmark', 'qgw', 'sql', '{ "statement": "select server_domain as domain, round(avg(tcp_rtt_ms),0) avg_rtt_latency_ms from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') and notEmpty(server_domain) group by server_domain order by avg_rtt_latency_ms desc limit 100", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('duplicate-logs-assessment', 'benchmark', 'qgw', 'sql', '{ "statement": "select ''Session Records'' as type, count(*) as num from(select log_id,count(*) as num from session_record where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') group by log_id having num >1) union all select ''Proxy Events'' as type, count(*) as num from (select log_id,count(*) as num from proxy_event where recv_time >= UNIX_TIMESTAMP(''${start_time}'') and recv_time <UNIX_TIMESTAMP(''${end_time}'') group by log_id having num >1)", "execution_mode": "oneshot" }',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-spectrum-summary', 'traffic_sketch_metric', 'qgw', 'dsl', '{"name":"traffic-spectrum-summary","filter":"vsys_id in (${vsys_id}) AND (${filter})","intervals":["${start_time}/${end_time}"]}',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-spectrum-unique-client-and-server-ips', 'traffic_sketch_metric', 'qgw', 'dsl', '{"name":"traffic-spectrum-unique-client-and-server-ips","filter":"vsys_id in (${vsys_id}) AND (${filter})","intervals":["${start_time}/${end_time}"]}',null); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-spectrum-app-distribution', 'traffic_sketch_metric', 'qgw', 'dsl', '{"name":"traffic-spectrum-app-distribution","filter":"app in (''${app}'') AND vsys_id in (${vsys_id}) AND (${filter})","intervals":["${start_time}/${end_time}"]}','The traffic usage of specified applications. Include the Top Server IP and Top Server Domain. '); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-spectrum-client-ip-connect-application-usage', 'traffic_sketch_metric', 'qgw', 'dsl', '{"name":"traffic-spectrum-client-ip-connect-application-usage","filter":"vsys_id in (${vsys_id}) AND (${filter})","intervals":["${start_time}/${end_time}"]}','The traffic usage of Internal Client IP to external application and External Client IP to Internal Application.'); +INSERT INTO `dataset` (`identifier_name`, `category`, `backend_engine`, `type`, `template`, `description`) VALUES ('traffic-spectrum-network-throughput-trend', 'traffic_sketch_metric', 'qgw', 'dsl', '{"name":"traffic-spectrum-network-throughput-trend","filter":"vsys_id in (${vsys_id}) AND (${filter})","intervals":["${start_time}/${end_time}"]}',null); COMMIT; SET FOREIGN_KEY_CHECKS = 1;
\ No newline at end of file diff --git a/config/flyway/tsg/V23.12__init_saved_query_job.sql b/config/flyway/tsg/V23.12__init_saved_query_job.sql index fdfff050..b0c7ca1d 100644 --- a/config/flyway/tsg/V23.12__init_saved_query_job.sql +++ b/config/flyway/tsg/V23.12__init_saved_query_job.sql @@ -4,8 +4,7 @@ SET FOREIGN_KEY_CHECKS = 0; -- ---------------------------- -- Table structure for saved_query_job -- ---------------------------- -DROP TABLE IF EXISTS `saved_query_job`; -CREATE TABLE `saved_query_job` ( +CREATE TABLE IF NOT EXISTS `saved_query_job` ( `job_id` varchar(128) NOT NULL COMMENT 'Job unique identifier', `query_sql` text NOT NULL COMMENT 'Query SQL', `state` varchar(16) NOT NULL COMMENT 'Enums: PENDING or RUNNING or DONE', diff --git a/config/flyway/tsg/V24.07__init_sys_storage_event.sql b/config/flyway/tsg/V24.07__init_sys_storage_event.sql new file mode 100644 index 00000000..af20799b --- /dev/null +++ b/config/flyway/tsg/V24.07__init_sys_storage_event.sql @@ -0,0 +1,22 @@ +/* + Sys Storage Event +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; +-- ---------------------------- +-- Table structure for dataset +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `sys_storage_event` ( + `id` int(64) NOT NULL AUTO_INCREMENT, + `log_type` varchar(512) NOT NULL COMMENT 'Enums: Traffic Logs or Metrics or Files', + `data_center` varchar(512) NOT NULL, + `used_size` bigint NOT NULL, + `total_allocated_size` bigint NOT NULL, + `bytes` bigint NOT NULL, + `since_time` bigint NOT NULL, + `generated_time` bigint NOT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +SET FOREIGN_KEY_CHECKS = 1;
\ No newline at end of file diff --git a/config/hazelcast.yaml b/config/hazelcast.yaml new file mode 100644 index 00000000..23863652 --- /dev/null +++ b/config/hazelcast.yaml @@ -0,0 +1,14 @@ +hazelcast: + cluster-name: qgw-cluster + network: + join: + multicast: + enabled: true + #tcp-ip: + # enabled: true + # member-list: + # - 192.168.44.12 + # - 192.168.44.13 + map: + adHocCache: + time-to-live-seconds: 300
\ No newline at end of file diff --git a/config/log4j2.xml b/config/log4j2.xml index 07564980..69928e64 100644 --- a/config/log4j2.xml +++ b/config/log4j2.xml @@ -12,7 +12,7 @@ <!--日志文件名称--> <property name="LOG_FILE_NAME">galaxy-qgw-service</property> <!--日志打印格式--> - <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ssZ}] [%p] [Thread:%t] %l %x - %m%n</property> + <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ssZ}] [%p] [Thread:%t] [%X{traceId}] %l %x - %m%n</property> </Properties> <appenders> diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_ip_dynamic_attribute.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_ip_dynamic_attribute.json index 715aeee3..fb0159e5 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_ip_dynamic_attribute.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_ip_dynamic_attribute.json @@ -24,7 +24,7 @@ { "name":"port", "label":"Port", - "type":"int" + "type":"long" }, { "name":"l7_protocol", diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_security_event.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_security_event.json index 71abfc96..865ffe57 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_security_event.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_security_event.json @@ -42,7 +42,7 @@ { "name":"is_builtin", "label":"IS Builtin", - "type":"int" + "type":"long" }, { "name":"severity", @@ -55,11 +55,61 @@ "type":"string" }, { + "name":"offender_country_region", + "label":"Offender Country", + "type":"string" + }, + { + "name":"offender_super_admin_area", + "label":"Offender Regin", + "type":"string" + }, + { + "name":"offender_admin_area", + "label":"Offender City", + "type":"string" + }, + { + "name":"offender_longitude", + "label":"Offender Longitude", + "type":"double" + }, + { + "name":"offender_latitude", + "label":"Offender Latitude", + "type":"double" + }, + { "name":"victim_ip", "label":"Victim IP", "type":"string" }, { + "name":"victim_country_region", + "label":"Victim Country", + "type":"string" + }, + { + "name":"victim_super_admin_area", + "label":"Victim Regin", + "type":"string" + }, + { + "name":"victim_admin_area", + "label":"Victim City", + "type":"string" + }, + { + "name":"victim_longitude", + "label":"Victim Longitude", + "type":"double" + }, + { + "name":"victim_latitude", + "label":"Victim Latitude", + "type":"double" + }, + { "name":"domain", "label":"Domain", "type":"string" @@ -81,7 +131,7 @@ }, { "name":"duration_s", - "label":"Duration (s)", + "label":"Duration(s)", "type":"long" }, { @@ -92,7 +142,7 @@ { "name":"status", "label":"Status", - "type":"int" + "type":"long" }, { "name":"event_info", diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/sql_datasets_variables.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/datasets_variables.json index 3bc1ef92..3bba531b 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/sql_datasets_variables.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/datasets_variables.json @@ -1,10 +1,10 @@ [ { - "key": "start", + "key": "start_time", "default": "2023-08-01 00:00:00+08:00" }, { - "key": "end", + "key": "end_time", "default": "2023-08-02T00:00:00+08:00" }, { @@ -36,6 +36,10 @@ "key": "alignment_period" }, { + "default": 300, + "key": "duration" + }, + { "key": "alignment_period_second", "default": 86400 }, diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/galaxy-qgw-service.yml b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/galaxy-qgw-service.yml index f68a3fcc..d3b6154e 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/galaxy-qgw-service.yml +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/galaxy-qgw-service.yml @@ -27,6 +27,8 @@ management: health: /component_status #MySql configuration spring: + sleuth: + enabled: true datasource: driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://192.168.44.55:3306/cn_olap?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=UTC&failOverReadOnly=false&connectTimeout=10000&socketTimeout=30000 @@ -49,8 +51,8 @@ hbase: zookeeperZnodeParent: /hbase rpcTimeout: 60000 report: - dbname: tsg - tableName: report_result + dbname: tsg_galaxy + tableName: saved_query_result columnFamily: response columnName: result ##############动态参数配置(修改后不需要重启项目)############## @@ -68,15 +70,12 @@ project: name: darnell url: email: -switch: - version: - ## 目前ck支持的版本：vsys_id、log_id，默认为vsys_id - schema: vsys_id ## ClickhHouse configuration clickhouse: url: http://192.168.44.55:8123 dbname: cyber_narrator_galaxy enableApproximateOptimizer: false + convertDefaultTimezoneIntoUTC: false realTimeAccount: username: ENC(Ao/e/87KQ8vOabP81PV/i4f/wvgztdKW) pin: ENC(KtRdLd84bYGGgk/Uv3+lUQFSxBfQ9ctJ) @@ -90,6 +89,7 @@ druid: url: 127.0.0.1:8082/druid/v2/sql dbname: druid socketTimeOut: 120000 + maxSubqueryRows: 2147483647 ## xxl-job-admin configuration xxl-job-admin: url: http://192.168.44.55:8181/xxl-job-admin @@ -99,19 +99,14 @@ xxl-job-admin: engine: maxCacheNum: 1048575 defaultResultNum: 100000 -service: - entity: - activeClientIP: 10000 - topServerIP: - TCPBySession: 100000 - TCPByUniqClientIP: 100000 - UDPBySession: 100000 - UDPByUniqClientIP: 40000 - topSNI: - defaultSize: 30000 - dataset: 2000000 - gtpc: - defaultSize: 100000 + fieldDiscovery: + topK: 100 + packetCombine: + delaySeconds: 5 + trafficSpectrum: + clientIPAppResultNum: 100 + highLatency: + threshold: 60000 ## ArangoDB configuration arango: server: http://192.168.44.55:8529 @@ -125,16 +120,16 @@ arango: ## hos cfg hos: uri: http://192.168.44.55:9098/hos - bucket: knowledge_base_hos_bucket + bucket: knowledge_base_bucket token: c21f969b5f03d33d43e04f8f136e7682 multiUploadStartSize: 5368709120 multiUploadPartSize: 10485760 ## knowledge value 格式 name:版本 knowledge: ipLocation: - userDefined: ip_user_defined:latest - builtIn: ip_builtin:latest - asn: asn_builtin:latest + userDefined: 004390bc-3135-4a6f-a492-3662ecb9e289_latest + builtIn: 64af7077-eb9b-4b8f-80cf-2ceebc89bea9_latest + asn: f9f6bc91-2142-4673-8249-e097c00fe1ea_latest ## http pool config http: pool: @@ -152,7 +147,7 @@ http: job: timeSlicing: enabled: true - interval: 1800 #单位: Second + interval: 3600 #单位: Second longPolling: enabled: true interactive: @@ -161,13 +156,18 @@ job: timeout: 3000 execution: timeout: 300000 - pool: - corePoolSize: 20 - maxPoolSize: 30 - queueCapacity: 10 + threadPool: + lightWeight: + corePoolSize: 10 + maxPoolSize: 20 + queueCapacity: 100 + heavyResource: + corePoolSize: 1 + maxPoolSize: 2 + queueCapacity: 80 ## task cfg task: pool: corePoolSize: 8 maxPoolSize: 16 - queueCapacity: 1000 + queueCapacity: 1000
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/location_subscriber.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/location_subscriber.json new file mode 100644 index 00000000..9faf4d8d --- /dev/null +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/location_subscriber.json @@ -0,0 +1,74 @@ +{ + "type": "record", + "name": "location_subscriber", + "namespace": "cyber_narrator_galaxy", + "doc": { + "partition_key": "stat_time", + "index_key": [ + "stat_time", + "subscriber_id" + ] + }, + "fields": [ + { + "name":"subscriber_id", + "label":"Subscriber ID", + "type":"string" + }, + { + "name":"imei", + "label":"IMEI", + "type":"string" + }, + { + "name":"imsi", + "label":"IMSI", + "type":"string" + }, + { + "name":"phone_number", + "label":"Phone Number", + "type":"string" + }, + { + "name":"apn", + "label":"APN", + "type":"string" + }, + { + "name":"subscriber_longitude", + "label":"Subscriber Longitude", + "type":"double" + }, + { + "name":"subscriber_latitude", + "label":"Subscriber Latitude", + "type":"double" + }, + { + "name":"first_location", + "label":"First Location", + "type":"string" + }, + { + "name":"second_location", + "label":"Second Location", + "type":"string" + }, + { + "name":"third_location", + "label":"Third Location", + "type":"string" + }, + { + "name":"data_source", + "label":"Data Source", + "type":"string" + }, + { + "name":"stat_time", + "label":"Stat Time", + "type":"long" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/meta_data.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/meta_data.json index 3c20b0c1..79d2f859 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/meta_data.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/meta_data.json @@ -5,7 +5,7 @@ "group": "CLICKHOUSE_GROUP", "tables": [ "session_record_cn", - "cn_entity_relation", + "session_relation_domain_ip_app", "cn_ip_dynamic_attribute", "cn_security_event", "metric_ip", @@ -22,28 +22,25 @@ "metric_dns_rcode", "metric_dns_rr_a", "metric_dns_rr_aaaa", - "metric_dns_rr_cname" + "metric_dns_rr_cname", + "metric_tag", + "location_subscriber", + "session_relation_subscriber_app" ] }, { "namespace": "system", "group": "CLICKHOUSE_GROUP", "tables": [ - "processes", + "processes_cluster", "tables_cluster" ] }, { - "namespace":"tsg", - "group":"HBASE_GROUP", - "tables":[ - "report_result" - ] - }, - { "namespace": "tsg_galaxy", "group": "HBASE_GROUP", "tables": [ + "saved_query_result" ] } ] diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_application.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_application.json index d1f49e82..fd3d3d28 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_application.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_application.json @@ -105,78 +105,78 @@ }, { "name":"tcp_c2s_lost_bytes_ratio", - "label":"Sequence Gap Loss (c2s)", - "type":"float" + "label":"Sequence Gap Loss(c2s)", + "type":"double" }, { "name":"tcp_s2c_lost_bytes_ratio", - "label":"Sequence Gap Loss (s2c)", - "type":"float" + "label":"Sequence Gap Loss(s2c)", + "type":"double" }, { "name":"tcp_lost_bytes_ratio", "label":"Sequence Gap Loss", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_o3_pkts_ratio", - "label":"Unordered Packets (c2s)", - "type":"float" + "label":"Unordered Packets(c2s)", + "type":"double" }, { "name":"tcp_s2c_o3_pkts_ratio", - "label":"Unordered Packets (s2c)", - "type":"float" + "label":"Unordered Packets(s2c)", + "type":"double" }, { "name":"tcp_o3_pkts_ratio", "label":"Unordered Packets", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_bytes_ratio", - "label":"Byte Retransmission (c2s)", - "type":"float" + "label":"Byte Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_bytes_ratio", - "label":"Byte Retransmission (s2c)", - "type":"float" + "label":"Byte Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_bytes_ratio", "label":"Byte Retransmission", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_pkts_ratio", - "label":"Packet Retransmission (c2s)", - "type":"float" + "label":"Packet Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_pkts_ratio", - "label":"Packet Retransmission (s2c)", - "type":"float" + "label":"Packet Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_pkts_ratio", "label":"Packet Retransmission", - "type":"float" + "type":"double" }, { "name":"avg_tcp_rtt_ms", - "label":"AVG TCP Handshake Latency (ms)", - "type":"float" + "label":"AVG TCP Handshake Latency(ms)", + "type":"double" }, { "name":"avg_http_response_latency_ms", - "label":"AVG HTTP.Response Latency (ms)", - "type":"float" + "label":"AVG HTTP.Response Latency(ms)", + "type":"double" }, { "name":"avg_ssl_handshake_latency_ms", - "label":"AVG SSL.Handshake Latency (ms)", - "type":"float" + "label":"AVG SSL.Handshake Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_asn.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_asn.json index 714c637e..0d46e824 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_asn.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_asn.json @@ -98,78 +98,78 @@ }, { "name":"tcp_c2s_lost_bytes_ratio", - "label":"Sequence Gap Loss (c2s)", - "type":"float" + "label":"Sequence Gap Loss(c2s)", + "type":"double" }, { "name":"tcp_s2c_lost_bytes_ratio", - "label":"Sequence Gap Loss (s2c)", - "type":"float" + "label":"Sequence Gap Loss(s2c)", + "type":"double" }, { "name":"tcp_lost_bytes_ratio", "label":"Sequence Gap Loss", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_o3_pkts_ratio", - "label":"Unordered Packets (c2s)", - "type":"float" + "label":"Unordered Packets(c2s)", + "type":"double" }, { "name":"tcp_s2c_o3_pkts_ratio", - "label":"Unordered Packets (s2c)", - "type":"float" + "label":"Unordered Packets(s2c)", + "type":"double" }, { "name":"tcp_o3_pkts_ratio", "label":"Unordered Packets", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_bytes_ratio", - "label":"Byte Retransmission (c2s)", - "type":"float" + "label":"Byte Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_bytes_ratio", - "label":"Byte Retransmission (s2c)", - "type":"float" + "label":"Byte Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_bytes_ratio", "label":"Byte Retransmission", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_pkts_ratio", - "label":"Packet Retransmission (c2s)", - "type":"float" + "label":"Packet Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_pkts_ratio", - "label":"Packet Retransmission (s2c)", - "type":"float" + "label":"Packet Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_pkts_ratio", "label":"Packet Retransmission", - "type":"float" + "type":"double" }, { "name":"avg_tcp_rtt_ms", - "label":"AVG TCP Handshake Latency (ms)", - "type":"float" + "label":"AVG TCP Handshake Latency(ms)", + "type":"double" }, { "name":"avg_http_response_latency_ms", - "label":"AVG HTTP.Response Latency (ms)", - "type":"float" + "label":"AVG HTTP.Response Latency(ms)", + "type":"double" }, { "name":"avg_ssl_handshake_latency_ms", - "label":"AVG SSL.Handshake Latency (ms)", - "type":"float" + "label":"AVG SSL.Handshake Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_qname.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_qname.json index 6dcf2b50..91440366 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_qname.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_qname.json @@ -57,8 +57,8 @@ }, { "name":"avg_response_latency_ms", - "label":"Avg Response Latency (ms)", - "type":"float" + "label":"Avg Response Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_qtype.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_qtype.json index 3208389c..77efddda 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_qtype.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_qtype.json @@ -13,7 +13,7 @@ { "name":"qtype", "label":"QTYPE", - "type":"int" + "type":"long" }, { "name":"stat_time", @@ -47,8 +47,8 @@ }, { "name":"avg_response_latency_ms", - "label":"Avg Response Latency (ms)", - "type":"float" + "label":"Avg Response Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rcode.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rcode.json index 09f3679d..47602d37 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rcode.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rcode.json @@ -13,7 +13,7 @@ { "name":"rcode", "label":"RCODE", - "type":"int" + "type":"long" }, { "name":"stat_time", @@ -47,8 +47,8 @@ }, { "name":"avg_response_latency_ms", - "label":"Avg Response Latency (ms)", - "type":"float" + "label":"Avg Response Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_a.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_a.json index 69586389..0458f92a 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_a.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_a.json @@ -47,8 +47,8 @@ }, { "name":"avg_response_latency_ms", - "label":"Avg Response Latency (ms)", - "type":"float" + "label":"Avg Response Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_aaaa.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_aaaa.json index 8db5c8af..fca5f2d3 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_aaaa.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_aaaa.json @@ -47,8 +47,8 @@ }, { "name":"avg_response_latency_ms", - "label":"Avg Response Latency (ms)", - "type":"float" + "label":"Avg Response Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_cname.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_cname.json index 8b7f6ef8..2f30330e 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_cname.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_rr_cname.json @@ -47,8 +47,8 @@ }, { "name":"avg_response_latency_ms", - "label":"Avg Response Latency (ms)", - "type":"float" + "label":"Avg Response Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_server_ip.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_server_ip.json index 45cdc3e2..5247cc6f 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_server_ip.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_dns_server_ip.json @@ -76,8 +76,8 @@ }, { "name":"avg_response_latency_ms", - "label":"Avg Response Latency (ms)", - "type":"float" + "label":"Avg Response Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_domain.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_domain.json index ddb3ff60..33514724 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_domain.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_domain.json @@ -104,78 +104,78 @@ }, { "name":"tcp_c2s_lost_bytes_ratio", - "label":"Sequence Gap Loss (c2s)", - "type":"float" + "label":"Sequence Gap Loss(c2s)", + "type":"double" }, { "name":"tcp_s2c_lost_bytes_ratio", - "label":"Sequence Gap Loss (s2c)", - "type":"float" + "label":"Sequence Gap Loss(s2c)", + "type":"double" }, { "name":"tcp_lost_bytes_ratio", "label":"Sequence Gap Loss", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_o3_pkts_ratio", - "label":"Unordered Packets (c2s)", - "type":"float" + "label":"Unordered Packets(c2s)", + "type":"double" }, { "name":"tcp_s2c_o3_pkts_ratio", - "label":"Unordered Packets (s2c)", - "type":"float" + "label":"Unordered Packets(s2c)", + "type":"double" }, { "name":"tcp_o3_pkts_ratio", "label":"Unordered Packets", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_bytes_ratio", - "label":"Byte Retransmission (c2s)", - "type":"float" + "label":"Byte Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_bytes_ratio", - "label":"Byte Retransmission (s2c)", - "type":"float" + "label":"Byte Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_bytes_ratio", "label":"Byte Retransmission", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_pkts_ratio", - "label":"Packet Retransmission (c2s)", - "type":"float" + "label":"Packet Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_pkts_ratio", - "label":"Packet Retransmission (s2c)", - "type":"float" + "label":"Packet Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_pkts_ratio", "label":"Packet Retransmission", - "type":"float" + "type":"double" }, { "name":"avg_tcp_rtt_ms", - "label":"AVG TCP Handshake Latency (ms)", - "type":"float" + "label":"AVG TCP Handshake Latency(ms)", + "type":"double" }, { "name":"avg_http_response_latency_ms", - "label":"AVG HTTP.Response Latency (ms)", - "type":"float" + "label":"AVG HTTP.Response Latency(ms)", + "type":"double" }, { "name":"avg_ssl_handshake_latency_ms", - "label":"AVG SSL.Handshake Latency (ms)", - "type":"float" + "label":"AVG SSL.Handshake Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_ip.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_ip.json index 74588e30..457dda8e 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_ip.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_ip.json @@ -97,78 +97,78 @@ }, { "name":"tcp_c2s_lost_bytes_ratio", - "label":"Sequence Gap Loss (c2s)", - "type":"float" + "label":"Sequence Gap Loss(c2s)", + "type":"double" }, { "name":"tcp_s2c_lost_bytes_ratio", - "label":"Sequence Gap Loss (s2c)", - "type":"float" + "label":"Sequence Gap Loss(s2c)", + "type":"double" }, { "name":"tcp_lost_bytes_ratio", "label":"Sequence Gap Loss", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_o3_pkts_ratio", - "label":"Unordered Packets (c2s)", - "type":"float" + "label":"Unordered Packets(c2s)", + "type":"double" }, { "name":"tcp_s2c_o3_pkts_ratio", - "label":"Unordered Packets (s2c)", - "type":"float" + "label":"Unordered Packets(s2c)", + "type":"double" }, { "name":"tcp_o3_pkts_ratio", "label":"Unordered Packets", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_bytes_ratio", - "label":"Byte Retransmission (c2s)", - "type":"float" + "label":"Byte Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_bytes_ratio", - "label":"Byte Retransmission (s2c)", - "type":"float" + "label":"Byte Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_bytes_ratio", "label":"Byte Retransmission", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_pkts_ratio", - "label":"Packet Retransmission (c2s)", - "type":"float" + "label":"Packet Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_pkts_ratio", - "label":"Packet Retransmission (s2c)", - "type":"float" + "label":"Packet Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_pkts_ratio", "label":"Packet Retransmission", - "type":"float" + "type":"double" }, { "name":"avg_tcp_rtt_ms", - "label":"AVG TCP Handshake Latency (ms)", - "type":"float" + "label":"AVG TCP Handshake Latency(ms)", + "type":"double" }, { "name":"avg_http_response_latency_ms", - "label":"AVG HTTP.Response Latency (ms)", - "type":"float" + "label":"AVG HTTP.Response Latency(ms)", + "type":"double" }, { "name":"avg_ssl_handshake_latency_ms", - "label":"AVG SSL.Handshake Latency (ms)", - "type":"float" + "label":"AVG SSL.Handshake Latency(ms)", + "type":"double" }, { "name":"asymmetric_sessions", diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_link.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_link.json index deee58cc..e1a7b224 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_link.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_link.json @@ -52,12 +52,12 @@ { "name":"out_link_id", "label":"Outgoing Link ID", - "type":"int" + "type":"long" }, { "name":"in_link_id", "label":"Incoming Link ID", - "type":"int" + "type":"long" }, { "name":"out_link_direction", @@ -141,78 +141,78 @@ }, { "name":"tcp_c2s_lost_bytes_ratio", - "label":"Sequence Gap Loss (c2s)", - "type":"float" + "label":"Sequence Gap Loss(c2s)", + "type":"double" }, { "name":"tcp_s2c_lost_bytes_ratio", - "label":"Sequence Gap Loss (s2c)", - "type":"float" + "label":"Sequence Gap Loss(s2c)", + "type":"double" }, { "name":"tcp_lost_bytes_ratio", "label":"Sequence Gap Loss", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_o3_pkts_ratio", - "label":"Unordered Packets (c2s)", - "type":"float" + "label":"Unordered Packets(c2s)", + "type":"double" }, { "name":"tcp_s2c_o3_pkts_ratio", - "label":"Unordered Packets (s2c)", - "type":"float" + "label":"Unordered Packets(s2c)", + "type":"double" }, { "name":"tcp_o3_pkts_ratio", "label":"Unordered Packets", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_bytes_ratio", - "label":"Byte Retransmission (c2s)", - "type":"float" + "label":"Byte Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_bytes_ratio", - "label":"Byte Retransmission (s2c)", - "type":"float" + "label":"Byte Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_bytes_ratio", "label":"Byte Retransmission", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_pkts_ratio", - "label":"Packet Retransmission (c2s)", - "type":"float" + "label":"Packet Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_pkts_ratio", - "label":"Packet Retransmission (s2c)", - "type":"float" + "label":"Packet Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_pkts_ratio", "label":"Packet Retransmission", - "type":"float" + "type":"double" }, { "name":"avg_tcp_rtt_ms", - "label":"AVG TCP Handshake Latency (ms)", - "type":"float" + "label":"AVG TCP Handshake Latency(ms)", + "type":"double" }, { "name":"avg_http_response_latency_ms", - "label":"AVG HTTP.Response Latency (ms)", - "type":"float" + "label":"AVG HTTP.Response Latency(ms)", + "type":"double" }, { "name":"avg_ssl_handshake_latency_ms", - "label":"AVG SSL.Handshake Latency (ms)", - "type":"float" + "label":"AVG SSL.Handshake Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_protocol.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_protocol.json index b1092393..afa24789 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_protocol.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_protocol.json @@ -19,7 +19,7 @@ { "name":"server_port", "label":"Server Port", - "type":"int" + "type":"long" }, { "name":"stat_time", @@ -93,78 +93,78 @@ }, { "name":"tcp_c2s_lost_bytes_ratio", - "label":"Sequence Gap Loss (c2s)", - "type":"float" + "label":"Sequence Gap Loss(c2s)", + "type":"double" }, { "name":"tcp_s2c_lost_bytes_ratio", - "label":"Sequence Gap Loss (s2c)", - "type":"float" + "label":"Sequence Gap Loss(s2c)", + "type":"double" }, { "name":"tcp_lost_bytes_ratio", "label":"Sequence Gap Loss", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_o3_pkts_ratio", - "label":"Unordered Packets (c2s)", - "type":"float" + "label":"Unordered Packets(c2s)", + "type":"double" }, { "name":"tcp_s2c_o3_pkts_ratio", - "label":"Unordered Packets (s2c)", - "type":"float" + "label":"Unordered Packets(s2c)", + "type":"double" }, { "name":"tcp_o3_pkts_ratio", "label":"Unordered Packets", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_bytes_ratio", - "label":"Byte Retransmission (c2s)", - "type":"float" + "label":"Byte Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_bytes_ratio", - "label":"Byte Retransmission (s2c)", - "type":"float" + "label":"Byte Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_bytes_ratio", "label":"Byte Retransmission", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_pkts_ratio", - "label":"Packet Retransmission (c2s)", - "type":"float" + "label":"Packet Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_pkts_ratio", - "label":"Packet Retransmission (s2c)", - "type":"float" + "label":"Packet Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_pkts_ratio", "label":"Packet Retransmission", - "type":"float" + "type":"double" }, { "name":"avg_tcp_rtt_ms", - "label":"AVG TCP Handshake Latency (ms)", - "type":"float" + "label":"AVG TCP Handshake Latency(ms)", + "type":"double" }, { "name":"avg_http_response_latency_ms", - "label":"AVG HTTP.Response Latency (ms)", - "type":"float" + "label":"AVG HTTP.Response Latency(ms)", + "type":"double" }, { "name":"avg_ssl_handshake_latency_ms", - "label":"AVG SSL.Handshake Latency (ms)", - "type":"float" + "label":"AVG SSL.Handshake Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_region.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_region.json index 1c6ab8ac..a7605163 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_region.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_region.json @@ -104,78 +104,78 @@ }, { "name":"tcp_c2s_lost_bytes_ratio", - "label":"Sequence Gap Loss (c2s)", - "type":"float" + "label":"Sequence Gap Loss(c2s)", + "type":"double" }, { "name":"tcp_s2c_lost_bytes_ratio", - "label":"Sequence Gap Loss (s2c)", - "type":"float" + "label":"Sequence Gap Loss(s2c)", + "type":"double" }, { "name":"tcp_lost_bytes_ratio", "label":"Sequence Gap Loss", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_o3_pkts_ratio", - "label":"Unordered Packets (c2s)", - "type":"float" + "label":"Unordered Packets(c2s)", + "type":"double" }, { "name":"tcp_s2c_o3_pkts_ratio", - "label":"Unordered Packets (s2c)", - "type":"float" + "label":"Unordered Packets(s2c)", + "type":"double" }, { "name":"tcp_o3_pkts_ratio", "label":"Unordered Packets", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_bytes_ratio", - "label":"Byte Retransmission (c2s)", - "type":"float" + "label":"Byte Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_bytes_ratio", - "label":"Byte Retransmission (s2c)", - "type":"float" + "label":"Byte Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_bytes_ratio", "label":"Byte Retransmission", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_pkts_ratio", - "label":"Packet Retransmission (c2s)", - "type":"float" + "label":"Packet Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_pkts_ratio", - "label":"Packet Retransmission (s2c)", - "type":"float" + "label":"Packet Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_pkts_ratio", "label":"Packet Retransmission", - "type":"float" + "type":"double" }, { "name":"avg_tcp_rtt_ms", - "label":"AVG TCP Handshake Latency (ms)", - "type":"float" + "label":"AVG TCP Handshake Latency(ms)", + "type":"double" }, { "name":"avg_http_response_latency_ms", - "label":"AVG HTTP.Response Latency (ms)", - "type":"float" + "label":"AVG HTTP.Response Latency(ms)", + "type":"double" }, { "name":"avg_ssl_handshake_latency_ms", - "label":"AVG SSL.Handshake Latency (ms)", - "type":"float" + "label":"AVG SSL.Handshake Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_subscriber_app.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_subscriber_app.json index 83e74522..6527d81e 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_subscriber_app.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_subscriber_app.json @@ -42,16 +42,6 @@ "type":"string" }, { - "name":"subscriber_longitude", - "label":"Subscriber Longitude", - "type":"string" - }, - { - "name":"subscriber_latitude", - "label":"Subscriber Latitude", - "type":"string" - }, - { "name":"stat_time", "label":"Stat Time", "type":"long" @@ -123,78 +113,78 @@ }, { "name":"tcp_c2s_lost_bytes_ratio", - "label":"Sequence Gap Loss (c2s)", - "type":"float" + "label":"Sequence Gap Loss(c2s)", + "type":"double" }, { "name":"tcp_s2c_lost_bytes_ratio", - "label":"Sequence Gap Loss (s2c)", - "type":"float" + "label":"Sequence Gap Loss(s2c)", + "type":"double" }, { "name":"tcp_lost_bytes_ratio", "label":"Sequence Gap Loss", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_o3_pkts_ratio", - "label":"Unordered Packets (c2s)", - "type":"float" + "label":"Unordered Packets(c2s)", + "type":"double" }, { "name":"tcp_s2c_o3_pkts_ratio", - "label":"Unordered Packets (s2c)", - "type":"float" + "label":"Unordered Packets(s2c)", + "type":"double" }, { "name":"tcp_o3_pkts_ratio", "label":"Unordered Packets", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_bytes_ratio", - "label":"Byte Retransmission (c2s)", - "type":"float" + "label":"Byte Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_bytes_ratio", - "label":"Byte Retransmission (s2c)", - "type":"float" + "label":"Byte Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_bytes_ratio", "label":"Byte Retransmission", - "type":"float" + "type":"double" }, { "name":"tcp_c2s_rtx_pkts_ratio", - "label":"Packet Retransmission (c2s)", - "type":"float" + "label":"Packet Retransmission(c2s)", + "type":"double" }, { "name":"tcp_s2c_rtx_pkts_ratio", - "label":"Packet Retransmission (s2c)", - "type":"float" + "label":"Packet Retransmission(s2c)", + "type":"double" }, { "name":"tcp_rtx_pkts_ratio", "label":"Packet Retransmission", - "type":"float" + "type":"double" }, { "name":"avg_tcp_rtt_ms", - "label":"AVG TCP Handshake Latency (ms)", - "type":"float" + "label":"AVG TCP Handshake Latency(ms)", + "type":"double" }, { "name":"avg_http_response_latency_ms", - "label":"AVG HTTP.Response Latency (ms)", - "type":"float" + "label":"AVG HTTP.Response Latency(ms)", + "type":"double" }, { "name":"avg_ssl_handshake_latency_ms", - "label":"AVG SSL.Handshake Latency (ms)", - "type":"float" + "label":"AVG SSL.Handshake Latency(ms)", + "type":"double" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_tag.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_tag.json new file mode 100644 index 00000000..8b46e708 --- /dev/null +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/metric_tag.json @@ -0,0 +1,44 @@ +{ + "type": "record", + "name": "metric_tag", + "namespace": "cyber_narrator_galaxy", + "doc": { + "partition_key": "stat_time", + "index_key": [ + "stat_time", + "tag" + ] + }, + "fields": [ + { + "name":"tag", + "label":"Tag", + "type":"string" + }, + { + "name":"stat_time", + "label":"Stat Time", + "type":"long" + }, + { + "name":"ip_sketch", + "label":"IP Sketch", + "type":"string" + }, + { + "name":"domain_sketch", + "label":"Domain Sketch", + "type":"string" + }, + { + "name":"ip_sketch_agg_state", + "label":"IP Sketch Aggregation State", + "type":"string" + }, + { + "name":"domain_sketch_agg_state", + "label":"Domain Sketch Aggregation State", + "type":"string" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/processes.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/processes.json deleted file mode 100644 index ddef00fd..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/processes.json +++ /dev/null @@ -1,11 +0,0 @@ -{
- "namespace": "system",
- "type": "record",
- "name": "processes",
- "fields": [
- {
- "name": "query_id",
- "type": "string"
- }
- ]
-}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/public_schema_info.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/public_schema_info.json index 5cd13e43..3e0895d2 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/public_schema_info.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/public_schema_info.json @@ -1,5 +1,177 @@ { "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)", + "metric_type": "counter", + "unit": "short" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "COUNT_DISTINCT(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)", + "metric_type": "counter", + "unit": "short" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "MEDIAN", + "label": "MEDIAN", + "function": "MEDIAN(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "MEDIAN_HDR", + "label": "MEDIAN_HDR", + "function": "MEDIAN_HDR(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "QUANTILE", + "label": "QUANTILE", + "function": "QUANTILE(expr, level)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P95_PERCENTILE", + "label": "P95_PERCENTILE", + "function": "QUANTILE(expr, 0.95)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P99_PERCENTILE", + "label": "P99_PERCENTILE", + "function": "QUANTILE(expr, 0.99)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "QUANTILE_HDR", + "label": "QUANTILE_HDR", + "function": "QUANTILE_HDR(expr, level)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P95_PERCENTILE_HDR", + "label": "P95_PERCENTILE_HDR", + "function": "QUANTILE_HDR(expr, 0.95)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P99_PERCENTILE_HDR", + "label": "P99_PERCENTILE_HDR", + "function": "QUANTILE_HDR(expr, 0.99)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "PERCENTILES_HDR", + "label": "PERCENTILES_HDR", + "function": "PERCENTILES_HDR(expr)", + "metric_type": "histogram", + "unit": "short" + }, + { + "name": "APPROX_COUNT_DISTINCT_HLLD", + "label": "COUNT_DISTINCT_HLLD", + "function": "APPROX_COUNT_DISTINCT_HLLD(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "APPROX_COUNT_DISTINCT_DS_HLL", + "label": "COUNT_DISTINCT_DS_HLL", + "function": "APPROX_COUNT_DISTINCT_DS_HLL(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "RATE", + "label": "RATE", + "function": "RATE(<expr>, <duration>, [interval])", + "metric_type": "gauge", + "unit": "sps" + }, + { + "name": "BITRATE", + "label": "BITRATE", + "function": "RATE(<expr>, <duration>, [interval]) * 8", + "metric_type": "gauge", + "unit": "bps" + } + ], + "date": [ + { + "name": "UNIX_TIMESTAMP", + "label": "UNIX_TIMESTAMP", + "function": "UNIX_TIMESTAMP(expr)" + }, + { + "name": "UNIX_TIMESTAMP_MILLIS", + "label": "UNIX_TIMESTAMP_MILLIS", + "function": "UNIX_TIMESTAMP_MILLIS(expr)" + }, + { + "name": "FROM_UNIXTIME", + "label": "FROM_UNIXTIME", + "function": "FROM_UNIXTIME(expr)" + }, + { + "name": "FROM_UNIXTIME_MILLIS", + "label": "FROM_UNIXTIME_MILLIS", + "function": "FROM_UNIXTIME_MILLIS(expr)" + }, + { + "name": "DATE_FORMAT", + "label": "DATE_FORMAT", + "function": "DATE_FORMAT(expr, format)" + }, + { + "name": "CONVERT_TZ", + "label": "CONVERT_TZ", + "function": "CONVERT_TZ(expr, from_tz, to_tz)" + }, + { + "name": "TIME_FLOOR_WITH_FILL", + "label": "TIME_FLOOR_WITH_FILL", + "function": "TIME_FLOOR_WITH_FILL(expr, period, fill)" + } + ], "operator": [ { "name": "=", @@ -32,6 +204,11 @@ "function": "expr <= value" }, { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { "name": "in", "label": "IN", "function": "expr in (values)" @@ -50,11 +227,72 @@ "name": "not like", "label": "NOT LIKE", "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + }, + { + "name": "bitAnd", + "label": "Bitwise AND", + "function": "bitAnd(expr, value) = value" } ] }, "schema_query": { "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "datetime", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "unix_timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "array", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "bit", + "functions": "COUNT,COUNT_DISTINCT" + } + ], "operator": [ { "type": "int", @@ -81,12 +319,24 @@ "functions": "=,!=,>,<,>=,<=" }, { + "type": "dateTime", + "functions": "=,!=,>,<,>=,<=" + }, + { "type": "timestamp", "functions": "=,!=,>,<,>=,<=" }, { "type": "unix_timestamp", "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has,notEmpty,empty" + }, + { + "type": "bit", + "functions": "=,!=,bitAnd" } ] } diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/report_result.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/saved_query_result.json index fdd1b7bd..8391dc6d 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/report_result.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/saved_query_result.json @@ -1,7 +1,7 @@ { "type": "record", - "name": "report_result", - "namespace": "tsg", + "name": "saved_query_result", + "namespace": "tsg_galaxy", "fields": [ { "name":"ROWKEY", diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/session_record_cn.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/session_record_cn.json index 18542c81..dc4157f1 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/session_record_cn.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/session_record_cn.json @@ -27,16 +27,22 @@ { "name":"start_timestamp_ms", "label":"Start Time", - "type":"long" + "type": { + "type": "string", + "logicalType": "datetime64" + } }, { "name":"end_timestamp_ms", "label":"End Time", - "type":"long" + "type": { + "type": "string", + "logicalType": "datetime64" + } }, { "name":"duration_ms", - "label":"Duration (ms)", + "label":"Duration(ms)", "type":"long" }, { @@ -57,12 +63,12 @@ { "name":"client_port", "label":"Client Port", - "type":"int" + "type":"long" }, { "name":"server_port", "label":"Server Port", - "type":"int" + "type":"long" }, { "name":"app", @@ -70,16 +76,21 @@ "type":"string" }, { - "name":"app_path", + "name":"app_transition", "label":"Application Path", "type":"string" }, { - "name":"protocol_path", + "name":"decoded_path", "label":"Protocol Path", "type":"string" }, { + "name":"ip_protocol", + "label":"IP Protocol", + "type":"string" + }, + { "name":"l7_protocol", "label":"L7 Protocol", "type":"string" @@ -87,12 +98,12 @@ { "name":"out_link_id", "label":"Outgoing Link ID", - "type":"int" + "type":"long" }, { "name":"in_link_id", "label":"Incoming Link ID", - "type":"int" + "type":"long" }, { "name":"subscriber_id", @@ -146,47 +157,47 @@ }, { "name":"tcp_c2s_lost_bytes", - "label":"Sequence Gap Loss (c2s)", + "label":"Sequence Gap Loss(c2s)", "type":"long" }, { "name":"tcp_s2c_lost_bytes", - "label":"Sequence Gap Loss (s2c)", + "label":"Sequence Gap Loss(s2c)", "type":"long" }, { "name":"tcp_c2s_o3_pkts", - "label":"Unordered Packets (c2s)", + "label":"Unordered Packets(c2s)", "type":"long" }, { "name":"tcp_s2c_o3_pkts", - "label":"Unordered Packets (s2c)", + "label":"Unordered Packets(s2c)", "type":"long" }, { "name":"tcp_c2s_rtx_bytes", - "label":"Byte Retransmission (c2s)", + "label":"Byte Retransmission(c2s)", "type":"long" }, { "name":"tcp_s2c_rtx_bytes", - "label":"Byte Retransmission (s2c)", + "label":"Byte Retransmission(s2c)", "type":"long" }, { "name":"tcp_c2s_rtx_pkts", - "label":"Packet Retransmission (c2s)", + "label":"Packet Retransmission(c2s)", "type":"long" }, { "name":"tcp_s2c_rtx_pkts", - "label":"Packet Retransmission (s2c)", - "type":"string" + "label":"Packet Retransmission(s2c)", + "type":"long" }, { "name":"tcp_rtt_ms", - "label":"TCP Handshake Latency (ms)", + "label":"TCP Handshake Latency(ms)", "type":"long" }, { @@ -227,7 +238,7 @@ { "name":"http_status_code", "label":"HTTP.Status Code", - "type":"int" + "type":"long" }, { "name":"ssl_version", @@ -267,22 +278,22 @@ { "name":"dns_qr", "label":"DNS.QR", - "type":"int" + "type":"long" }, { "name":"dns_opcode", "label":"DNS.OPCODE", - "type":"int" + "type":"long" }, { "name":"dns_aa", "label":"DNS.AA", - "type":"int" + "type":"long" }, { "name":"dns_rcode", "label":"DNS.RCODE", - "type":"int" + "type":"long" }, { "name":"dns_qname", @@ -292,17 +303,17 @@ { "name":"dns_qtype", "label":"DNS.QTYPE", - "type":"int" + "type":"long" }, { "name":"dns_qclass", "label":"DNS.QCLASS", - "type":"int" + "type":"long" }, { "name":"dns_sub", "label":"DNS.SUB", - "type":"int" + "type":"long" }, { "name":"dns_rr", @@ -346,7 +357,7 @@ }, { "name":"ssh_kex_alg", - "label":"SSH. Key Exchange Algorithm", + "label":"SSH.Key Exchange Algorithm", "type":"string" }, { @@ -567,18 +578,18 @@ }, { "name":"http_response_latency_ms", - "label":"HTTP.Response Latency (ms)", - "type":"int" + "label":"HTTP.Response Latency(ms)", + "type":"long" }, { "name":"ssl_handshake_latency_ms", - "label":"SSL.Handshake Latency (ms)", - "type":"int" + "label":"SSL.Handshake Latency(ms)", + "type":"long" }, { "name":"dns_response_latency_ms", - "label":"DNS.Response Latency (ms)", - "type":"int" + "label":"DNS.Response Latency(ms)", + "type":"long" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_entity_relation.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/session_relation_domain_ip_app.json index 3e739e9b..037aa456 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/cn_entity_relation.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/session_relation_domain_ip_app.json @@ -1,6 +1,6 @@ { "type": "record", - "name": "cn_entity_relation", + "name": "session_relation_domain_ip_app", "namespace": "cyber_narrator_galaxy", "doc": { "partition_key": "create_time", diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/session_relation_subscriber_app.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/session_relation_subscriber_app.json new file mode 100644 index 00000000..a7824282 --- /dev/null +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/session_relation_subscriber_app.json @@ -0,0 +1,64 @@ +{ + "type": "record", + "name": "session_relation_subscriber_app", + "namespace": "cyber_narrator_galaxy", + "doc": { + "partition_key": "create_time", + "index_key": [ + "subscriber_id", + "app_name" + ] + }, + "fields": [ + { + "name":"create_time", + "label":"Create Time", + "type":"long" + }, + { + "name":"update_time", + "label":"Update Time", + "type":"long" + }, + { + "name":"app_name", + "label":"App.Name", + "type":"string" + }, + { + "name":"app_category", + "label":"App.Category", + "type":"string" + }, + { + "name":"app_subcategory", + "label":"App.Subcategory", + "type":"string" + }, + { + "name":"subscriber_id", + "label":"Subscriber ID", + "type":"string" + }, + { + "name":"imei", + "label":"IMEI", + "type":"string" + }, + { + "name":"imsi", + "label":"IMSI", + "type":"string" + }, + { + "name":"phone_number", + "label":"Phone Number", + "type":"string" + }, + { + "name":"apn", + "label":"APN", + "type":"string" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/version.json b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/version.json index 733c3c5b..343eb63f 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/version.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/version.json @@ -1,95 +1,68 @@ { - "product": "185 Environment", - "version": "21.05", - "registered": "Geedge", - "updated": "2020-08-01 00:00:00", - "components": { - "oss": [ - { - "name": "zookeeper", - "version": "3.4.9", - "licenseType": "Apache License 2.0", - "description": "分布式应用程序协调服务" - }, - { - "name": "kafka", - "version": "2.11_1.0.0", - "licenseType": "Apache License 2.0", - "description": "消息队列" - }, - { - "name": "habse", - "version": "2.2.3", - "licenseType": "Apache License 2.0", - "description": "用于文件系统和存储Radius数据" - }, - { - "name": "flume", - "version": "1.9.0", - "licenseType": "Apache License 2.0", - "description": "日志补全传输" - }, - { - "name": "clickhouse", - "version": "19.17.4.11", - "licenseType": "Apache License 2.0", - "description": "原始日志数据库" - }, - { - "name": "druid", - "version": "0.16.0", - "licenseType": "Apache License 2.0", - "description": "分析实时数据并提供低延迟查询的OLAP应用程序" - }, - { - "name": "gohangout", - "version": "1.14.4.20200616", - "description": "动态获取原始日志表schema入库程序" - } - ], - "apps": [ - { - "name": "galaxy-qgw-service", - "version": "v3.2.200730", - "description": "数据平台对外统一查询网关" - }, - { - "name": "galaxy-report-service", - "version": "20.07.08", - "description": "自定义报表查询服务" - }, - { - "name": "galaxy-hos-service", - "version": "20.07.17", - "description": "对象存储服务" - }, - { - "name": "xxl-job-admin", - "version": "v1.0.2.20200506", - "description": "分布式任务调度平台" - }, - { - "name": "xxl-job", - "version": "v1.0.2.20200628", - "description": "分布式任务调度平台-执行器" - } - ], - "tasks": [ - { - "name": "flume", - "version": "flume-config-20.08", - "description": "原始日志补全、subscriber更新、Radius上下线功能" - }, - { - "name": "druid", - "version": "druid-config-20.08", - "description": "所有分析日志任务" - }, - { - "name": "gohangout", - "version": "gohangout-config-20.08", - "description": "原始日志入库、上下线日志入库" - } - ] - } + "product": "CN OLAP", + "version": "24.01", + "registered": "Geedge Networks", + "updated": "2024-01-31 12:00:00", + "components": [ + { + "name": "zookeeper", + "version": "3.5.9", + "description": "分布式应用程序协调服务" + }, + { + "name": "kafka", + "version": "3.4.1", + "description": "消息队列" + }, + { + "name": "Hadoop", + "version": "2.7.1", + "description": "分布式文件系统用于HBase、Druid等组件深度存储" + }, + { + "name": "hbase", + "version": "2.2.3", + "description": "用于文件系统和存储Radius数据" + }, + { + "name": "Flink", + "version": "1.13.1", + "description": "流数据计算框架用于日志预处理及部分统计" + }, + { + "name": "clickHouse", + "version": "21.8.13.1", + "description": "OLAP Analytic storage" + }, + { + "name": "druid", + "version": "26.0.0", + "description": "分析实时数据并提供低延迟查询的OLAP应用程序" + }, + { + "name":"nacos", + "version": "2.0.2", + "description": "分布式配置中心" + }, + { + "name": "hos", + "version": "20.07.17", + "description": "对象存储服务" + }, + { + "name": "jobAdmin", + "version": "2.2.0", + "description": "分布式任务调度平台" + }, + { + "name": "mariaDB", + "version": "10.5.3", + "description": "传统数据库用于nacos/druid/galaxy-job-service/dataset数据存储" + }, + { + "name":"arango", + "version": "3.6.4", + "description": "图数据库用于存储IPlearning统计结果" + } + ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/active_defence_event.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/active_defence_event.json deleted file mode 100644 index ca10319a..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/active_defence_event.json +++ /dev/null @@ -1,382 +0,0 @@ -{ - "type": "record", - "name": "active_defence_event", - "namespace": "tsg_galaxy_v3", - "doc": { - "primary_key": "common_log_id", - "partition_key": "common_recv_time", - "index_key": { - "$ref": "public_schema_info_2311.json#/active_defence_event/index_key" - }, - "schema_query": { - "dimensions": [ - "common_policy_id", - "ad_target_ip", - "ad_cc_target_url" - ], - "metrics": [ - "ad_target_ip", - "ad_sent_byte_num", - "ad_sent_pkt_num", - "ad_cc_initiate_connection_num", - "ad_cc_established_connection_num", - "ad_cc_rejected_connection_num" - ], - "filters": [ - "common_policy_id", - "ad_target_ip", - "ad_target_port", - "ad_protocol", - "common_address_type", - "ad_sent_byte_num", - "ad_sent_pkt_num", - "ad_cc_initiate_connection_num", - "ad_cc_established_connection_num", - "ad_cc_rejected_connection_num" - ] - }, - "schema_type": { - "REFLECTION": { - "columns": [ - "common_recv_time", - "common_log_id", - "common_policy_id", - "common_address_type", - "common_device_id", - "common_out_link_id", - "common_in_link_id", - "common_entrance_id", - "common_user_region", - "ad_method", - "ad_protocol", - "ad_target_ip", - "ad_target_port", - "ad_target_ip_location", - "ad_target_ip_asn", - "ad_reflector_profile_id", - "ad_sent_pkt_num", - "ad_sent_byte_num", - "ad_generate_time" - ], - "default_columns": [ - "common_recv_time", - "common_log_id", - "common_policy_id", - "ad_target_ip", - "ad_target_port", - "ad_reflector_profile_id", - "ad_sent_pkt_num", - "ad_sent_byte_num" - ] - }, - "FLOOD": { - "columns": [ - "common_recv_time", - "common_log_id", - "common_policy_id", - "common_address_type", - "common_device_id", - "common_out_link_id", - "common_in_link_id", - "common_entrance_id", - "common_user_region", - "ad_method", - "ad_protocol", - "ad_target_ip", - "ad_target_port", - "ad_target_ip_location", - "ad_target_ip_asn", - "ad_claimed_src_ip_profile_id", - "ad_sent_pkt_num", - "ad_sent_byte_num", - "ad_generate_time" - ], - "default_columns": [ - "common_recv_time", - "common_log_id", - "common_policy_id", - "ad_target_ip", - "ad_target_port", - "ad_claimed_src_ip_profile_id", - "ad_protocol" - ] - }, - "CC": { - "columns": [ - "common_recv_time", - "common_log_id", - "common_policy_id", - "common_address_type", - "common_device_id", - "common_out_link_id", - "common_in_link_id", - "common_entrance_id", - "common_user_region", - "ad_method", - "ad_protocol", - "ad_cc_target_url", - "ad_claimed_src_ip_profile_id", - "ad_cc_initiate_connection_num", - "ad_cc_established_connection_num", - "ad_cc_rejected_connection_num", - "ad_generate_time" - ], - "default_columns": [ - "common_recv_time", - "common_log_id", - "common_policy_id", - "ad_cc_target_url", - "ad_claimed_src_ip_profile_id", - "ad_protocol" - ] - } - }, - "default_columns": [ - "common_recv_time", - "common_log_id", - "common_policy_id", - "ad_target_ip", - "ad_target_port", - "ad_cc_target_url" - ] - }, - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "format": [ - { - "function": "current_timestamp" - } - ], - "visibility": "enabled" - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_log_id", - "label": "Log ID", - "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], - "visibility": "enabled" - }, - "type": "long" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "doc": { - "visibility": "enabled" - }, - "type": "long" - }, - { - "name": "common_address_type", - "label": "Address Type", - "doc": { - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "doc": { - "visibility": "disabled" - }, - "type": "int" - }, - { - "name": "common_device_id", - "label": "Device ID", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "common_out_link_id", - "label": "Outgoing Link ID", - "doc": { - "visibility": "hidden" - }, - "type": "int" - }, - { - "name": "common_in_link_id", - "label": "Incoming Link ID", - "doc": { - "visibility": "hidden" - }, - "type": "int" - }, - { - "name": "common_user_region", - "label": "User Region", - "doc": { - "visibility": "hidden" - }, - "type": "string" - }, - { - "name": "ad_target_ip", - "label": "Target IP", - "doc": { - "constraints": { - "type": "ip" - }, - "format": [ - { - "function": "geo_ip_country", - "appendTo": "ad_target_ip_location" - }, - { - "function": "geo_asn", - "appendTo": "ad_target_ip_asn" - } - ], - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "ad_target_port", - "label": "Target Port", - "doc": { - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "ad_cc_target_url", - "label": "Target URL", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "ad_target_ip_location", - "label": "Target Location", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "ad_target_ip_asn", - "label": "Target ASN", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "ad_protocol", - "label": "Protocol", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "ad_method", - "label": "Method", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "ad_claimed_src_ip_profile_id", - "label": "Claimed Profile ID", - "doc": { - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "ad_reflector_profile_id", - "label": "Reflector Profile ID", - "doc": { - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "ad_sent_pkt_num", - "label": "Packets Sent", - "doc": { - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "ad_sent_byte_num", - "label": "Bytes Sent", - "doc": { - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "ad_cc_initiate_connection_num", - "label": "Initiate Numbers", - "doc": { - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "ad_cc_established_connection_num", - "label": "Established Numbers", - "doc": { - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "ad_cc_rejected_connection_num", - "label": "Rejected Numbers", - "doc": { - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "ad_generate_time", - "label": "Generate Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "enabled" - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/application_protocol_stat.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/application_protocol_stat.json index aa078265..5e0b3eb6 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/application_protocol_stat.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/application_protocol_stat.json @@ -8,21 +8,13 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "protocol_label", - "app_full_path" + "app_name" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "protocol_label", - "app_full_path", "sessions", "in_bytes", "out_bytes", @@ -31,54 +23,52 @@ "c2s_pkts", "s2c_pkts", "c2s_bytes", - "s2c_bytes", - "c2s_fragments", - "s2c_fragments", - "c2s_tcp_lost_bytes", - "s2c_tcp_lost_bytes", - "c2s_tcp_ooorder_pkts", - "s2c_tcp_ooorder_pkts", - "c2s_tcp_retransmitted_pkts", - "s2c_tcp_retransmitted_pkts", - "c2s_tcp_retransmitted_bytes", - "s2c_tcp_retransmitted_bytes", - "client_ip_sketch" + "s2c_bytes" ], "filters": [ - "vsys_id", - "device_id", "device_group", "data_center", - "protocol_label", - "app_full_path", - "sessions", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts", - "c2s_pkts", - "s2c_pkts", - "c2s_bytes", - "s2c_bytes", - "c2s_fragments", - "s2c_fragments", - "c2s_tcp_lost_bytes", - "s2c_tcp_lost_bytes", - "c2s_tcp_ooorder_pkts", - "s2c_tcp_ooorder_pkts", - "c2s_tcp_retransmitted_pkts", - "s2c_tcp_retransmitted_pkts", - "c2s_tcp_retransmitted_bytes", - "s2c_tcp_retransmitted_bytes", - "client_ip_sketch" + "app_name" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 15, - "ingestion_delay": 15 + "ingestion_period": "PT1S", + "compaction_period": "PT1M", + "skip_compaction_offset_from_latest": "P7D", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "in_bytes": "bytes", + "out_bytes": "bytes", + "in_pkts": "packets", + "out_pkts": "packets", + "c2s_bytes": "bytes", + "s2c_bytes": "bytes", + "c2s_pkts": "packets", + "s2c_pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "in_pkts": "pps", + "out_pkts": "pps", + "c2s_bytes": "Bps", + "s2c_bytes": "Bps", + "c2s_pkts": "pps", + "s2c_pkts": "pps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps", + "c2s_bytes": "bps", + "s2c_bytes": "bps" + } + } } }, "fields": [ @@ -102,7 +92,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -158,7 +148,7 @@ "label": "App Name", "doc": { "constraints": { - "operator_functions": "=,in" + "operator_functions": "=,in,notEmpty,empty" }, "visibility": "enabled" }, @@ -168,26 +158,22 @@ "name": "sessions", "label": "Sessions", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" }, { - "name": "client_ip_sketch", - "label": "Client IP Sketch", - "doc": { - "visibility": "enabled" - }, - "type": { - "type": "array", - "items": "string", - "logicalType": "array" - } - }, - { "name": "c2s_pkts", "label": "Packets Sent", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -196,6 +182,10 @@ "name": "s2c_pkts", "label": "Packets Received", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -204,6 +194,10 @@ "name": "c2s_bytes", "label": "Bytes Sent", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -212,86 +206,130 @@ "name": "s2c_bytes", "label": "Bytes Received", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "c2s_fragments", - "label": "Fragmentation Packets (c2s)", + "label": "Fragmentation Packets(c2s)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "s2c_fragments", - "label": "Fragmentation Packets (s2c)", + "label": "Fragmentation Packets(s2c)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "c2s_tcp_lost_bytes", - "label": "Sequence Gap Loss (c2s)", + "label": "Sequence Gap Loss(c2s)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "s2c_tcp_lost_bytes", - "label": "Sequence Gap Loss (s2c)", + "label": "Sequence Gap Loss(s2c)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "c2s_tcp_ooorder_pkts", - "label": "TCP out-of-order Packets (c2s)", + "label": "TCP out-of-order Packets(c2s)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "s2c_tcp_ooorder_pkts", - "label": "TCP out-of-order Packets (s2c)", + "label": "TCP out-of-order Packets(s2c)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "c2s_tcp_retransmitted_pkts", - "label": "TCP Retransmission Packets (c2s)", + "label": "TCP Retransmission Packets(c2s)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "s2c_tcp_retransmitted_pkts", - "label": "TCP Retransmission Packets (s2c)", + "label": "TCP Retransmission Packets(s2c)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "c2s_tcp_retransmitted_bytes", - "label": "TCP Retransmission Bytes (c2s)", + "label": "TCP Retransmission Bytes(c2s)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" }, { "name": "s2c_tcp_retransmitted_bytes", - "label": "TCP Retransmission Bytes (s2c)", + "label": "TCP Retransmission Bytes(s2c)", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -300,6 +338,10 @@ "name": "in_bytes", "label": "Incoming Bytes", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -308,6 +350,10 @@ "name": "in_pkts", "label": "Incoming Packets", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -316,6 +362,10 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -324,6 +374,10 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/assessment_event.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/assessment_event.json index e69803c7..c09a1ac0 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/assessment_event.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/assessment_event.json @@ -5,11 +5,19 @@ "doc": { "primary_key": "log_id", "partition_key": "recv_time", - "index_key": { - "$ref": "public_schema_info.json#/assessment_event/index_key" - }, + "index_key": [ + "vsys_id", + "recv_time", + "log_id" + ], "functions": { "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "time": [ + "recv_time", + "assessment_date" + ] } }, "fields": [ @@ -31,11 +39,9 @@ "name": "log_id", "label": "Log ID", "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], + "constraints": { + "aggregation_functions": "COUNT" + }, "visibility": "enabled" }, "type": "long" @@ -46,7 +52,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/datapath_telemetry_record.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/datapath_telemetry_record.json new file mode 100644 index 00000000..6e210df2 --- /dev/null +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/datapath_telemetry_record.json @@ -0,0 +1,197 @@ +{ + "type": "record", + "name": "datapath_telemetry_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "log_id", + "partition_key": "recv_time", + "index_key": [ + "vsys_id", + "job_id", + "recv_time", + "timestamp_us" + ], + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "time": [ + "recv_time", + "timestamp_us" + ] + } + }, + "fields": [ + { + "name": "recv_time", + "label": "Receive Time", + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled" + }, + "type": { + "type": "long", + "logicalType": "unix_timestamp" + } + }, + { + "name": "log_id", + "label": "Log ID", + "doc": { + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "vsys_id", + "label": "Vsys ID", + "doc": { + "visibility": "enabled" + }, + "type": "int", + "default": 1 + }, + { + "name": "timestamp_us", + "label": "Timestamp", + "doc": { + "constraints": { + "type": "unix_timestamp_us" + }, + "visibility": "enabled" + }, + "type": { + "type": "long", + "logicalType": "unix_timestamp_us" + } + }, + { + "name": "egress_action", + "label": "Egress Action", + "doc": { + "visibility": "enabled", + "data": [ + { + "code": 0, + "value": "send" + }, + { + "code": 1, + "value": "drop" + } + ] + }, + "type": { + "type": "int" + } + }, + { + "name": "job_id", + "label": "Job ID", + "doc": { + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "sled_ip", + "label": "Sled IP", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", + "type": "ip" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "traffic_link_id", + "label": "Traffic Link ID", + "doc": { + "visibility": "enabled" + }, + "type": "int" + }, + { + "name": "source_ip", + "label": "Source IP", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "source_port", + "label": "Source Port", + "doc": { + "visibility": "enabled" + }, + "type": "int" + }, + { + "name": "destination_ip", + "label": "Destination IP", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "destination_port", + "label": "DestinationPort", + "doc": { + "visibility": "enabled" + }, + "type": "int" + }, + { + "name": "packet", + "label": "Packet", + "doc": { + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "packet_length", + "label": "Packet Length", + "doc": { + "visibility": "enabled" + }, + "type": "int" + }, + { + "name": "measurements", + "label": "Measurements", + "doc": { + "visibility": "enabled" + }, + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/sql_datasets_variables.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/datasets_variables.json index 7d890db1..581dadd5 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/sql_datasets_variables.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/datasets_variables.json @@ -1,11 +1,11 @@ [ { - "key": "start", - "default": "2023-08-01 00:00:00" + "key": "start_time", + "default": "" }, { - "key": "end", - "default": "2023-08-02 00:00:00" + "key": "end_time", + "default": "" }, { "key": "vsys_id", @@ -28,6 +28,10 @@ "default": 1 }, { + "key": "object_type", + "default": "ip" + }, + { "key": "object_id", "default": 1 }, @@ -57,7 +61,7 @@ }, { "key": "metric", - "default": 1 + "default": "bytes" }, { "key": "sf_profile_id", @@ -85,7 +89,7 @@ }, { "key": "filter", - "default": "" + "default": "1 = 1" }, { "key": "timestampdiff_second", @@ -128,11 +132,43 @@ "default": "client_ip" }, { + "key": "column_name_long", + "default": "flags" + }, + { + "key": "column_name_string", + "default": "flags_identify_info" + }, + { + "key": "column_name_array", + "default": "fqdn_category_list" + }, + { "key": "source", - "default": "statistics_rule" + "default": "session_record" + }, + { + "key": "unix_timestamp_column", + "default": "recv_time" + }, + { + "key": "fn", + "default": "count" + }, + { + "key": "field_list", + "default": "log_id" + }, + { + "key": "duration", + "default": 300 + }, + { + "key": "job_id", + "default": "0b30c0db-e0e0-4679-9cde-fe62b44b29a7" }, { - "key": "timestamp_column", - "default": "__time" + "key": "app", + "default": "unknown" } ]
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/dos_event.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/dos_event.json index b9c55469..34acf1dc 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/dos_event.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/dos_event.json @@ -4,16 +4,24 @@ "namespace": "tsg_galaxy_v3", "doc": { "primary_key": "log_id", - "partition_key": "start_time", + "partition_key": "recv_time", "ttl": null, "default_ttl": 2592000, - "index_key": { - "$ref": "public_schema_info.json#/dos_event/index_key" - }, + "index_key": [ + "vsys_id", + "destination_ip", + "recv_time", + "log_id" + ], "functions": { "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "recv_time", + "start_time", + "end_time" + ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } @@ -31,14 +39,27 @@ "session_rate" ], "internal_columns": [ - "start_time", + "recv_time", "log_id" - ] + ], + "measurements": { + "field_discovery_metric": { + "sessions": [ + { + "fn": "count", + "column": "log_id", + "value": "sessions", + "label": "Sessions", + "unit": "sessions" + } + ] + } + } }, "fields": [ { - "name": "start_time", - "label": "Start Time", + "name": "recv_time", + "label": "Receive Time", "doc": { "constraints": { "type": "unix_timestamp" @@ -52,8 +73,8 @@ } }, { - "name": "end_time", - "label": "End Time", + "name": "start_time", + "label": "Start Time", "doc": { "constraints": { "type": "unix_timestamp" @@ -67,8 +88,8 @@ } }, { - "name": "recv_time", - "label": "Receive Time", + "name": "end_time", + "label": "End Time", "doc": { "constraints": { "type": "unix_timestamp" @@ -85,11 +106,9 @@ "name": "log_id", "label": "Log ID", "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], + "constraints": { + "aggregation_functions": "COUNT" + }, "visibility": "enabled", "ttl": null }, @@ -97,7 +116,16 @@ }, { "name": "profile_id", - "label": "Profile ID", + "label": "Profile ID(Deprecated)", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "rule_id", + "label": "Rule ID", "doc": { "visibility": "enabled", "ttl": null @@ -119,7 +147,7 @@ "label": "Attack Type", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -137,6 +165,14 @@ { "code": "DNS Flood", "value": "DNS Flood" + }, + { + "code": "NTP Flood", + "value": "NTP Flood" + }, + { + "code": "Custom Network Attack", + "value": "Custom Network Attack" } ], "visibility": "enabled", @@ -149,7 +185,7 @@ "label": "Severity", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -191,6 +227,10 @@ "name": "destination_ip", "label": "Destination IP", "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -207,7 +247,7 @@ }, { "name": "source_ip_list", - "label": "Source IPs", + "label": "Source IP List", "doc": { "visibility": "enabled", "ttl": null @@ -216,7 +256,7 @@ }, { "name": "source_country_list", - "label": "Source Countries", + "label": "Source Country List", "doc": { "visibility": "enabled", "ttl": null @@ -224,6 +264,15 @@ "type": "string" }, { + "name": "sessions", + "label": "Sessions", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { "name": "session_rate", "label": "Sessions/s", "doc": { @@ -236,6 +285,15 @@ "type": "long" }, { + "name": "packets", + "label": "Packets", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { "name": "packet_rate", "label": "Packets/s", "doc": { @@ -248,6 +306,15 @@ "type": "long" }, { + "name": "bytes", + "label": "Bytes", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { "name": "bit_rate", "label": "Bits/s", "doc": { diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_subscribers.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/dos_protection_rule_hits.json index f3cd0de0..165c2439 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_subscribers.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/dos_protection_rule_hits.json @@ -1,6 +1,6 @@ { "type": "record", - "name": "top_subscribers", + "name": "dos_protection_rule_hits", "namespace": "druid", "doc": { "partition_key": "__time", @@ -8,44 +8,53 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", + "rule_id", "device_group", - "data_center", - "subscriber_id" + "data_center" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "subscriber_id", - "sessions", + "hit_count", "in_bytes", "out_bytes", "in_pkts", "out_pkts" ], "filters": [ - "vsys_id", - "device_id", + "rule_id", "device_group", - "data_center", - "subscriber_id", - "sessions", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 300, - "ingestion_delay": 15 + "ingestion_period": "PT1S", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "hit_count": "sessions", + "in_bytes": "bytes", + "out_bytes": "bytes", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "hit_count": "sps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -64,25 +73,8 @@ } }, { - "name": "vsys_id", - "label": "Vsys ID", - "doc": { - "visibility": "enabled" - }, - "type": "int", - "default": 1 - }, - { - "name": "device_id", - "label": "Device ID", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "device_group", - "label": "Device Group", + "name": "data_center", + "label": "Data Center", "type": "string", "doc": { "constraints": { @@ -90,15 +82,15 @@ }, "data": { "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", - "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" }, "visibility": "enabled" } }, { - "name": "data_center", - "label": "Data Center", + "name": "device_group", + "label": "Device Group", "type": "string", "doc": { "constraints": { @@ -106,48 +98,60 @@ }, "data": { "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", - "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" }, "visibility": "enabled" } }, { - "name": "subscriber_id", - "label": "Subscriber ID", + "name": "device_id", + "label": "Device ID", "doc": { "visibility": "enabled" }, "type": "string" }, { - "name": "metric", - "label": "Metric", + "name": "vsys_id", + "label": "Vsys ID", "doc": { "visibility": "enabled" }, - "type": "string" + "type": "long", + "default": 1 }, { - "name": "sessions", - "label": "Sessions", + "name": "rule_id", + "label": "Rule", "doc": { + "constraints": { + "operator_functions": "=,in" + }, "visibility": "enabled" }, "type": "long" }, { - "name": "in_bytes", - "label": "Incoming Bytes", + "name": "hit_count", + "label": "Hit Count", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" }, { - "name": "in_pkts", - "label": "Incoming Packets", + "name": "in_bytes", + "label": "Incoming Bytes", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -156,6 +160,22 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -164,6 +184,10 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/dos_protection_rule_metric.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/dos_protection_rule_metric.json new file mode 100644 index 00000000..7b8807b9 --- /dev/null +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/dos_protection_rule_metric.json @@ -0,0 +1,209 @@ +{ + "type": "record", + "name": "dos_protection_rule_metric", + "namespace": "druid", + "doc": { + "partition_key": "__time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "time": [ + "__time" + ], + "dimensions": [ + "rule_id", + "device_group", + "data_center" + ], + "metrics": [ + "sessions", + "bytes", + "pkts" + ], + "filters": [ + "rule_id", + "device_group", + "data_center" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + } + }, + "measurements": { + "ingestion_period": "PT1S", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "bytes": "bytes", + "pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "bytes": "Bps", + "pkts": "pps" + }, + "BITRATE": { + "bytes": "bps" + } + } + } + }, + "fields": [ + { + "name": "__time", + "label": "Time", + "type": { + "type": "string", + "logicalType": "timestamp" + }, + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "enabled" + } + }, + { + "name": "data_center", + "label": "Data Center", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "visibility": "enabled" + } + }, + { + "name": "device_group", + "label": "Device Group", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "visibility": "enabled" + } + }, + { + "name": "device_id", + "label": "Device ID", + "doc": { + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "vsys_id", + "label": "Vsys ID", + "doc": { + "visibility": "enabled" + }, + "type": "long", + "default": 1 + }, + { + "name": "rule_id", + "label": "Rule", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "client_ip", + "label": "Client IP", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "client_country", + "label": "Client Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "server_ip", + "label": "Server IP", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "server_country", + "label": "Server Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "sessions", + "label": "Sessions", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "bytes", + "label": "Bytes", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/galaxy-qgw-service.yml b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/galaxy-qgw-service.yml index 32f0136c..4d8b3340 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/galaxy-qgw-service.yml +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/galaxy-qgw-service.yml @@ -27,6 +27,8 @@ management: health: /component_status #MySql configuration spring: + sleuth: + enabled: true datasource: driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://192.168.44.12:3306/tsg_olap?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=UTC&failOverReadOnly=false&connectTimeout=10000&socketTimeout=30000 @@ -49,8 +51,8 @@ hbase: zookeeperZnodeParent: /hbase rpcTimeout: 60000 report: - dbname: tsg - tableName: report_result + dbname: tsg_galaxy + tableName: saved_query_result columnFamily: response columnName: result ##############动态参数配置(修改后不需要重启项目)############## @@ -68,15 +70,12 @@ project: name: darnell url: email: -switch: - version: - ## 目前ck支持的版本：vsys_id、log_id，默认为vsys_id - schema: vsys_id ## ClickhHouse configuration clickhouse: url: http://192.168.44.12:8123 dbname: tsg_galaxy_v3 enableApproximateOptimizer: true + convertDefaultTimezoneIntoUTC: false realTimeAccount: username: ENC(hYFEuRJJWd93TZg5VbK/o3OXUmxI/irv) pin: ENC(qUA355VopKSx6kwwwXZwqWWEYSu76Slz) @@ -90,6 +89,7 @@ druid: url: 192.168.44.12:8089/druid/v2/sql dbname: druid socketTimeOut: 120000 + maxSubqueryRows: 2147483647 ## xxl-job-admin configuration xxl-job-admin: url: http://192.168.44.12:8181/xxl-job-admin @@ -99,22 +99,17 @@ xxl-job-admin: engine: maxCacheNum: 1048575 defaultResultNum: 100000 -service: - entity: - activeClientIP: 10000 - topServerIP: - TCPBySession: 100000 - TCPByUniqClientIP: 100000 - UDPBySession: 100000 - UDPByUniqClientIP: 40000 - topSNI: - defaultSize: 30000 - dataset: 2000000 - gtpc: - defaultSize: 100000 + fieldDiscovery: + topK: 100 + packetCombine: + delaySeconds: 5 + trafficSpectrum: + clientIPAppResultNum: 100 + highLatency: + threshold: 60000 ## ArangoDB configuration arango: - server: http://192.168.44.83:8529 + server: http://192.168.44.12:8529 database: tsg_galaxy_v3 username: ENC(UyZxsi4PT6kQXJAep5qvQQ==) pin: ENC(qUA355VopKSx6kwwwXZwqWWEYSu76Slz) @@ -125,16 +120,16 @@ arango: ## hos cfg hos: uri: http://192.168.44.12:9098/hos - bucket: knowledge_base_hos_bucket + bucket: knowledge_base_bucket token: c21f969b5f03d33d43e04f8f136e7682 multiUploadStartSize: 5368709120 multiUploadPartSize: 10485760 ## knowledge value 格式 name:版本 knowledge: ipLocation: - userDefined: ip_user_defined:latest - builtIn: ip_builtin:latest - asn: asn_builtin:latest + userDefined: 004390bc-3135-4a6f-a492-3662ecb9e289_latest + builtIn: 64af7077-eb9b-4b8f-80cf-2ceebc89bea9_latest + asn: f9f6bc91-2142-4673-8249-e097c00fe1ea_latest ## http pool config http: pool: @@ -145,14 +140,14 @@ http: per: route: 200 request: - timeout: 10000 + timeout: 30000 response: timeout: 120000 ## job cfg job: timeSlicing: enabled: true - interval: 1800 #单位: Second + interval: 3600 #单位: Second longPolling: enabled: true interactive: @@ -161,13 +156,18 @@ job: timeout: 3000 execution: timeout: 300000 - pool: - corePoolSize: 20 - maxPoolSize: 30 - queueCapacity: 10 + threadPool: + lightWeight: + corePoolSize: 10 + maxPoolSize: 20 + queueCapacity: 100 + heavyResource: + corePoolSize: 1 + maxPoolSize: 2 + queueCapacity: 80 ## task cfg task: pool: corePoolSize: 8 maxPoolSize: 16 - queueCapacity: 1000 + queueCapacity: 1000
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/gtpc_knowledge_base.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/gtpc_knowledge_base.json deleted file mode 100644 index 6cd5ccce..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/gtpc_knowledge_base.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "type": "record", - "name": "gtpc_knowledge_base", - "namespace": "tsg_galaxy", - "fields": [ - { - "name": "vsys_id", - "label": "System Id", - "type": "int" - }, - { - "name": "uplink_teid", - "label": "Up Link Teid", - "type": "int" - }, - { - "name": "downlink_teid", - "label": "Down Link Teid", - "type": "int" - }, - { - "name": "ROWKEY", - "label": "Row Key", - "type": "string" - }, - { - "name": "apn", - "label": "APN", - "type": "string" - }, - { - "name": "phone_number", - "label": "Phone Number", - "type": "string" - }, - { - "name": "imsi", - "label": "IMSI", - "type": "string" - }, - { - "name": "imei", - "label": "IMEI", - "type": "string" - }, - { - "name": "last_update_time", - "label": "Last Update Time", - "type": "int" - }, - { - "name": "msg_type", - "label": "Message Type", - "type": "int" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/gtpc_record.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/gtpc_record.json deleted file mode 100644 index 37f15dbf..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/gtpc_record.json +++ /dev/null @@ -1,1740 +0,0 @@ -{ - "type": "record", - "name": "gtpc_record", - "namespace": "tsg_galaxy_v3", - "doc": { - "primary_key": "common_log_id", - "partition_key": "common_recv_time", - "ttl": null, - "default_ttl": 2592000, - "index_key": { - "$ref": "public_schema_info_2311.json#/gtpc_record/index_key" - }, - "functions": { - "$ref": "public_schema_info_2311.json#/functions" - }, - "schema_query": { - "dimensions": [ - "common_stream_trace_id", - "common_stream_dir", - "common_start_time", - "common_end_time", - "common_start_timestamp_ms", - "common_end_timestamp_ms", - "common_processing_time", - "common_ingestion_time", - "common_device_id", - "common_data_center", - "common_sled_ip", - "common_device_group", - "common_address_type", - "common_schema_type", - "common_vsys_id", - "common_action", - "common_sub_action", - "common_policy_id", - "common_client_ip", - "common_internal_ip", - "common_client_port", - "common_client_os_name", - "common_client_location", - "common_client_asn", - "common_in_src_mac", - "common_out_src_mac", - "common_server_ip", - "common_external_ip", - "common_server_port", - "common_server_os_name", - "common_server_location", - "common_server_asn", - "common_server_fqdn", - "common_server_domain", - "common_in_dest_mac", - "common_out_dest_mac", - "common_app_full_path", - "common_app_identify_info", - "common_app_label", - "common_app_behavior", - "common_protocol_label", - "common_l4_protocol", - "common_tunnel_endpoint_a_desc", - "common_tunnel_endpoint_b_desc", - "gtp_version", - "gtp_apn", - "gtp_imei", - "gtp_imsi", - "gtp_phone_number", - "gtp_msg_type" - ], - "metrics": [ - "common_start_time", - "common_end_time", - "common_start_timestamp_ms", - "common_end_timestamp_ms", - "common_device_id", - "common_data_center", - "common_sled_ip", - "common_device_group", - "common_address_type", - "common_schema_type", - "common_vsys_id", - "common_action", - "common_sub_action", - "common_policy_id", - "common_client_ip", - "common_internal_ip", - "common_client_port", - "common_client_os_name", - "common_client_location", - "common_client_asn", - "common_subscriber_id", - "common_in_src_mac", - "common_out_src_mac", - "common_server_ip", - "common_external_ip", - "common_server_port", - "common_server_os_name", - "common_server_location", - "common_server_asn", - "common_server_fqdn", - "common_server_domain", - "common_in_dest_mac", - "common_out_dest_mac", - "common_app_full_path", - "common_app_identify_info", - "common_app_label", - "common_app_behavior", - "common_protocol_label", - "common_l4_protocol", - "common_c2s_pkt_num", - "common_s2c_pkt_num", - "common_c2s_byte_num", - "common_s2c_byte_num", - "common_sessions", - "common_con_duration_ms", - "common_establish_latency_ms", - "gtp_version", - "gtp_apn", - "gtp_imei", - "gtp_imsi", - "gtp_phone_number" - ], - "filters": [ - "common_stream_trace_id", - "common_stream_dir", - "common_start_time", - "common_end_time", - "common_start_timestamp_ms", - "common_end_timestamp_ms", - "common_con_duration_ms", - "common_establish_latency_ms", - "common_processing_time", - "common_ingestion_time", - "common_device_id", - "common_data_center", - "common_sled_ip", - "common_device_group", - "common_address_type", - "common_schema_type", - "common_vsys_id", - "common_action", - "common_sub_action", - "common_policy_id", - "common_client_ip", - "common_internal_ip", - "common_client_port", - "common_client_os_name", - "common_client_location", - "common_client_asn", - "common_subscriber_id", - "common_in_src_mac", - "common_out_src_mac", - "common_server_ip", - "common_external_ip", - "common_server_port", - "common_server_os_name", - "common_server_location", - "common_server_asn", - "common_server_fqdn", - "common_server_domain", - "common_in_dest_mac", - "common_out_dest_mac", - "common_app_full_path", - "common_app_identify_info", - "common_app_label", - "common_app_behavior", - "common_protocol_label", - "common_service_category", - "common_l4_protocol", - "common_tunnel_endpoint_a_desc", - "common_tunnel_endpoint_b_desc", - "common_c2s_pkt_num", - "common_s2c_pkt_num", - "common_c2s_byte_num", - "common_s2c_byte_num", - "gtp_version", - "gtp_apn", - "gtp_imei", - "gtp_imsi", - "gtp_phone_number", - "gtp_end_user_ipv4", - "gtp_end_user_ipv6", - "gtp_uplink_teid", - "gtp_downlink_teid", - "gtp_msg_type" - ], - "references": { - "$ref": "public_schema_info_2311.json#/schema_query/references" - }, - "details": { - "general": [ - "common_recv_time", - "common_log_id", - "common_stream_trace_id", - "common_address_type", - "common_schema_type", - "common_vsys_id", - "common_t_vsys_id", - "common_flags", - "common_flags_identify_info", - "common_direction", - "common_stream_dir", - "common_start_time", - "common_end_time", - "common_start_timestamp_ms", - "common_end_timestamp_ms", - "common_con_duration_ms", - "common_establish_latency_ms", - "common_processing_time", - "common_ingestion_time", - "common_entrance_id", - "common_device_id", - "common_out_link_id", - "common_in_link_id", - "common_isp", - "common_data_center", - "common_device_group", - "common_sled_ip" - ], - "source": [ - "common_client_ip", - "common_internal_ip", - "common_client_port", - "common_client_os_name", - "common_client_location", - "common_client_asn", - "common_subscriber_id", - "common_imei", - "common_imsi", - "common_phone_number", - "common_in_src_mac", - "common_out_src_mac" - ], - "destination": [ - "common_server_ip", - "common_external_ip", - "common_server_port", - "common_server_os_name", - "common_server_location", - "common_server_asn", - "common_server_fqdn", - "common_server_domain", - "common_in_dest_mac", - "common_out_dest_mac" - ], - "application": [ - "common_app_id", - "common_app_full_path", - "common_userdefine_app_name", - "common_app_identify_info", - "common_app_label", - "common_app_surrogate_id", - "common_l7_protocol", - "common_protocol_label", - "common_service_category", - "common_service", - "common_l4_protocol", - "common_app_behavior" - ], - "transmission": [ - "common_sessions", - "common_c2s_pkt_num", - "common_s2c_pkt_num", - "common_c2s_byte_num", - "common_s2c_byte_num", - "common_c2s_pkt_diff", - "common_s2c_pkt_diff", - "common_c2s_byte_diff", - "common_s2c_byte_diff", - "common_c2s_ipfrag_num", - "common_s2c_ipfrag_num", - "common_c2s_tcp_lostlen", - "common_s2c_tcp_lostlen", - "common_c2s_tcp_unorder_num", - "common_s2c_tcp_unorder_num", - "common_c2s_pkt_retrans", - "common_s2c_pkt_retrans", - "common_c2s_byte_retrans", - "common_s2c_byte_retrans", - "common_first_ttl", - "common_tcp_client_isn", - "common_tcp_server_isn", - "common_mirrored_pkts", - "common_mirrored_bytes" - ], - "other": [ - "common_device_tag", - "common_encapsulation", - "common_tunnels", - "common_address_list", - "common_has_dup_traffic", - "common_stream_error", - "common_link_info_c2s", - "common_link_info_s2c", - "common_packet_capture_file", - "common_tunnel_endpoint_a_desc", - "common_tunnel_endpoint_b_desc", - "common_action", - "common_sub_action", - "common_policy_id", - "common_user_tags", - "common_user_region", - "common_shaping_rule_ids" - ] - } - }, - "schema_type": { - "GTP-C": { - "$ref": "public_schema_info_2311.json#/schema_type/GTP-C" - } - }, - "default_columns": [ - "common_recv_time", - "common_log_id", - "gtp_version", - "gtp_msg_type", - "gtp_imsi", - "gtp_imei", - "gtp_phone_number", - "common_client_ip", - "common_server_ip" - ], - "internal_columns": [ - "common_recv_time", - "common_log_id", - "common_sessions", - "common_flags_identify_info", - "common_app_identify_info", - "common_packet_capture_file", - "common_tunnel_endpoint_a_desc", - "common_tunnel_endpoint_b_desc" - ], - "tunnel_type": { - "$ref": "public_schema_info_2311.json#/tunnel_type" - } - }, - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_log_id", - "label": "Log ID", - "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_subscriber_id", - "label": "Subscriber ID", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_imei", - "label": "IMEI", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_imsi", - "label": "IMSI", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_phone_number", - "label": "Phone Number", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_in_src_mac", - "label": "Incoming Source MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_out_src_mac", - "label": "Outgoing Source MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_client_ip", - "label": "Client IP", - "doc": { - "constraints": { - "type": "ip" - }, - "format": [ - { - "function": "geo_asn", - "appendTo": "common_client_asn" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_internal_ip", - "label": "Internal IP", - "doc": { - "constraints": { - "type": "ip" - }, - "format": [ - { - "function": "if", - "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_client_port", - "label": "Client Port", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_client_os_name", - "type": "string", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "label": "Client OS" - }, - { - "name": "common_l4_protocol", - "label": "L4 Protocol", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_address_type", - "label": "Address Type", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_server_ip", - "label": "Server IP", - "doc": { - "constraints": { - "type": "ip" - }, - "format": [ - { - "function": "geo_asn", - "appendTo": "common_server_asn" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_port", - "label": "Server Port", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_server_os_name", - "type": "string", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "label": "Server OS" - }, - { - "name": "common_external_ip", - "label": "External IP", - "doc": { - "constraints": { - "type": "ip" - }, - "format": [ - { - "function": "if", - "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_action", - "label": "Action", - "doc": { - "visibility": "hidden", - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "0", - "value": "None" - }, - { - "code": "1", - "value": "Monitor" - }, - { - "code": "2", - "value": "Intercept" - }, - { - "code": "16", - "value": "Deny" - }, - { - "code": "96", - "value": "Allow" - }, - { - "code": "128", - "value": "Allow(Deprecated)" - } - ], - "ttl": null - }, - "type": "int" - }, - { - "name": "common_direction", - "label": "Direction", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "69", - "value": "outbound" - }, - { - "code": "73", - "value": "inbound" - } - ], - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_sled_ip", - "label": "Sled IP", - "doc": { - "constraints": { - "type": "ip" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_client_location", - "label": "Client Location", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_client_asn", - "label": "Client ASN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_location", - "label": "Server Location", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_asn", - "label": "Server ASN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_fqdn", - "label": "Server FQDN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_domain", - "label": "Server Domain", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_in_dest_mac", - "label": "Incoming Destination MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_out_dest_mac", - "label": "Outgoing Destination MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_sessions", - "label": "Sessions", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_pkt_num", - "label": "Packets Sent", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_pkt_num", - "label": "Packets Received", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_byte_num", - "label": "Bytes Sent", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_byte_num", - "label": "Bytes Received", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_pkt_diff", - "label": "Packets Sent (Delta)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_pkt_diff", - "label": "Packets Received (Delta)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_byte_diff", - "label": "Bytes Sent (Delta)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_byte_diff", - "label": "Bytes Received (Delta)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_service", - "label": "Service", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_schema_type", - "label": "Schema Type", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "GTP-C", - "value": "GTP-C" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_vsys_id", - "label": "Vsys ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int", - "default": 1 - }, - { - "name": "common_t_vsys_id", - "label": "Traffic Vsys ID", - "doc": { - "allow_query": "false", - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_flags", - "label": "Flags", - "doc": { - "constraints": { - "type": "bit", - "operator_functions": "=,!=,bitAnd" - }, - "data": { - "$ref": "public_schema_info_2311.json#/fields/common_flags/data" - }, - "allow_query": "false", - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "bit" - } - }, - { - "name": "common_flags_identify_info", - "label": "Flags Identify Info", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_user_tags", - "label": "User Tags", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_sub_action", - "label": "Sub Action", - "doc": { - "data": [ - { - "code": "allow", - "value": "Allow" - }, - { - "code": "deny", - "value": "Deny" - }, - { - "code": "monitor", - "value": "Monitor" - }, - { - "code": "replace", - "value": "Replace" - }, - { - "code": "redirect", - "value": "Redirect" - }, - { - "code": "insert", - "value": "Insert" - }, - { - "code": "hijack", - "value": "Hijack" - } - ], - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_user_region", - "label": "User Region", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_shaping_rule_ids", - "label": "Shaping Rule IDs", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "int", - "logicalType": "array" - } - }, - { - "name": "common_device_id", - "label": "Device ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_out_link_id", - "label": "Outgoing Link ID", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_in_link_id", - "label": "Incoming Link ID", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_isp", - "label": "ISP", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_device_tag", - "label": "Device Tag", - "doc": { - "visibility": "hidden", - "format": [ - { - "function": "flattenSpec", - "appendTo": "common_data_center", - "param": "$.tags[?(@.tag=='data_center')][0].value" - }, - { - "function": "flattenSpec", - "appendTo": "common_device_group", - "param": "$.tags[?(@.tag=='device_group')][0].value" - } - ], - "ttl": null - }, - "type": "string" - }, - { - "name": "common_data_center", - "label": "Data Center", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", - "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_device_group", - "label": "Device Group", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", - "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_app_behavior", - "label": "Application Behavior", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_encapsulation", - "label": "Encapsulation", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "public_schema_info_2311.json#/fields/common_encapsulation/data" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_app_label", - "label": "Application Label", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_tunnels", - "label": "Tunnels", - "doc": { - "visibility": "enabled", - "ttl": null, - "format": [ - { - "function": "gtpc_match", - "appendTo": "common_imsi,common_imei,common_phone_number", - "param": "$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_a2b_teid,$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_b2a_teid" - } - ] - }, - "type": "string" - }, - { - "name": "common_protocol_label", - "label": "Protocol Label", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_app_id", - "label": "Application ID", - "type": "string", - "doc": { - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_app_full_path", - "label": "Application Full Path", - "type": "string", - "doc": { - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_userdefine_app_name", - "label": "User Define App Name", - "type": "string", - "doc": { - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_app_identify_info", - "label": "App Identity Info", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_app_surrogate_id", - "label": "Surrogate ID", - "type": "string", - "doc": { - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_l7_protocol", - "label": "L7 Protocol", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_service_category", - "label": "FQDN Category", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "disabled", - "dict_location": { - "path": "/v1/category/dict", - "key": "categoryId", - "value": "categoryName" - }, - "ttl": null - }, - "type": { - "type": "array", - "items": "int", - "logicalType": "array" - } - }, - { - "name": "common_start_time", - "label": "Start Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_end_time", - "label": "End Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_start_timestamp_ms", - "label": "Start Time", - "doc": { - "constraints": { - "operator_functions": "=,!=,>,<,>=,<=", - "type": "datetime64" - }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "string", - "logicalType": "datetime64" - } - }, - { - "name": "common_end_timestamp_ms", - "label": "End Time", - "doc": { - "constraints": { - "operator_functions": "=,!=,>,<,>=,<=", - "type": "datetime64" - }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "string", - "logicalType": "datetime64" - } - }, - { - "name": "common_establish_latency_ms", - "label": "TCP Handshake Latency (ms)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_con_duration_ms", - "label": "Duration (ms)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_stream_dir", - "label": "Stream Direction(Deprecated)", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "1", - "value": "c2s" - }, - { - "code": "2", - "value": "s2c" - }, - { - "code": "3", - "value": "double" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_address_list", - "label": "Address List", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_has_dup_traffic", - "label": "Duplication Traffic", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "public_schema_info_2311.json#/fields/common_has_dup_traffic/data" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_stream_error", - "label": "Stream Error", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_stream_trace_id", - "label": "Session ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info (c2s)", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info (s2c)", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_packet_capture_file", - "label": "Packet Capture File", - "doc": { - "visibility": "hidden", - "constraints": { - "type": "file" - }, - "ttl": null - }, - "type": "string" - }, - { - "name": "common_tunnel_endpoint_a_desc", - "label": "Tunnel Endpoint A Description", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_tunnel_endpoint_b_desc", - "label": "Tunnel Endpoint B Description", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_c2s_ipfrag_num", - "label": "Fragmentation Packets (c2s)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_ipfrag_num", - "label": "Fragmentation Packets (s2c)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_tcp_lostlen", - "label": "Sequence Gap Loss (c2s)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_tcp_lostlen", - "label": "Sequence Gap Loss (s2c)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_tcp_unorder_num", - "label": "Unordered Packets (c2s)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_tcp_unorder_num", - "label": "Unordered Packets (s2c)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_pkt_retrans", - "label": "Packet Retransmission (c2s)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_pkt_retrans", - "label": "Packet Retransmission (s2c)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_byte_retrans", - "label": "Byte Retransmission (c2s)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_byte_retrans", - "label": "Byte Retransmission (s2c)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_tcp_client_isn", - "label": "TCP Client ISN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_tcp_server_isn", - "label": "TCP Server ISN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_first_ttl", - "label": "First TTL", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_processing_time", - "label": "Processing Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "format": [ - { - "function": "current_timestamp" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_ingestion_time", - "label": "Ingestion Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "format": [ - { - "function": "ingestion_time" - }, - { - "function": "get_value", - "appendTo": "common_recv_time" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_mirrored_pkts", - "label": "Mirrored Packets", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_mirrored_bytes", - "label": "Mirrored Bytes", - "type": "long", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "gtp_version", - "label": "Version", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "gtp_apn", - "label": "APN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "gtp_imei", - "label": "IMEI", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "gtp_imsi", - "label": "IMSI", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "gtp_phone_number", - "label": "Phone Number", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "gtp_uplink_teid", - "label": "Uplink TEID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "gtp_downlink_teid", - "label": "Downlink TEID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "gtp_msg_type", - "label": "Message Type", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "create", - "value": "create" - }, - { - "code": "modify", - "value": "modify" - }, - { - "code": "delete", - "value": "delete" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "gtp_end_user_ipv4", - "label": "End User Address V4", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "gtp_end_user_ipv6", - "label": "End User Address V6", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/job_result.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/job_result.json deleted file mode 100644 index c5ff0cdc..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/job_result.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "type": "record", - "name": "job_result", - "namespace": "tsg_galaxy", - "fields": [ - { - "name": "ROWKEY", - "label": "Row Key", - "type": "string" - }, - { - "name": "is_failed", - "label": "Failed", - "type": "boolean" - }, - { - "name": "is_done", - "label": "Done", - "type": "boolean" - }, - { - "name": "is_canceled", - "label": "Canceled", - "type": "boolean" - }, - { - "name": "done_progress", - "label": "Progress", - "type": "double" - }, - { - "name": "last_query_time", - "label": "Last Query Time", - "type": "long" - }, - { - "name": "duration_time", - "label": "Duration Time", - "type": "long" - }, - { - "name": "count", - "label": "Count", - "type": "long" - }, - { - "name": "job_property", - "label": "Job Property", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/meta_data.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/meta_data.json index f0b3ea57..01091b8d 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/meta_data.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/meta_data.json @@ -4,19 +4,16 @@ "namespace": "tsg_galaxy_v3", "group": "CLICKHOUSE_GROUP", "tables": [ - "radius_onff_log", "session_record", "transaction_record", - "radius_record", "voip_record", - "gtpc_record", "security_event", "proxy_event", "dos_event", - "active_defence_event", - "sys_packet_capture_event", "assessment_event", - "monitor_event" + "monitor_event", + "datapath_telemetry_record", + "traffic_sketch_metric" ] }, { @@ -28,7 +25,7 @@ "columns_cluster", "disks_cluster", "parts_cluster", - "processes", + "processes_cluster", "query_log", "tables", "clusters", @@ -45,36 +42,28 @@ "top_external_ips", "top_server_domains", "top_server_fqdns", - "top_subscribers", + "top_client_countries", + "top_server_countries", "application_protocol_stat", "traffic_general_stat", - "traffic_top_destination_ip_metrics_log", "traffic_shaping_rule_hits", - "sys_storage_log", "service_chaining_rule_hits", "service_function_status", "security_rule_hits", "proxy_rule_hits", "statistics_rule", "object_statistics", - "statistics_rule_hits" - ] - }, - { - "namespace":"tsg", - "group":"HBASE_GROUP", - "tables":[ - "report_result" + "statistics_rule_hits", + "monitor_rule_hits", + "dos_protection_rule_hits", + "dos_protection_rule_metric" ] }, { "namespace": "tsg_galaxy", "group": "HBASE_GROUP", "tables": [ - "relation_account_framedip", - "recommendation_app_cip", - "job_result", - "gtpc_knowledge_base" + "saved_query_result" ] } ] diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/monitor_event.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/monitor_event.json index a4d7a904..c4014c94 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/monitor_event.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/monitor_event.json @@ -5,22 +5,31 @@ "doc": { "primary_key": "log_id", "partition_key": "recv_time", - "index_key": { - "$ref": "public_schema_info.json#/monitor_event/index_key" - }, + "index_key": [ + "vsys_id", + "security_action", + "proxy_action", + "decoded_as", + "data_center", + "device_group", + "recv_time" + ], "ttl": null, "default_ttl": 2592000, - "index_table": "", "functions": { "$ref": "public_schema_info.json#/functions" }, "schema_query": { - "dimensions": [ - "session_id", + "time": [ + "recv_time", "start_timestamp_ms", "end_timestamp_ms", "processing_time", "ingestion_time", + "insert_time" + ], + "dimensions": [ + "session_id", "device_id", "out_link_id", "in_link_id", @@ -29,14 +38,21 @@ "device_group", "address_type", "decoded_as", + "direction", "vsys_id", "flags", "flags_identify_info", + "c2s_ttl", + "s2c_ttl", "monitor_rule_list", "client_ip", "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", "imei", @@ -50,17 +66,25 @@ "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", "in_dest_mac", "out_dest_mac", - "app_path", + "app_transition", "app", + "app_category", "app_debug_info", "app_content", - "protocol_path", + "app_extra_info", + "ip_protocol", + "decoded_path", "dup_traffic_flag", + "tunnel_id_list", "tunnel_endpoint_a_desc", "tunnel_endpoint_b_desc", "http_url", @@ -96,6 +120,7 @@ "mail_attachment_name", "mail_attachment_name_charset", "mail_eml_file", + "mail_starttls_flag", "dns_message_id", "dns_qr", "dns_opcode", @@ -181,45 +206,49 @@ "rdp_certificate_count", "rdp_certificate_permanent", "rdp_encryption_level", - "rdp_encryption_method" + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "internal_port_list", + "external_port_list", + "internal_asn_list", + "external_asn_list", + "monitor_rule_id" ], "metrics": [ - "start_timestamp_ms", - "end_timestamp_ms", - "device_id", - "data_center", - "sled_ip", - "device_group", - "address_type", - "decoded_as", - "vsys_id", + "log_id", + "c2s_ttl", + "s2c_ttl", "client_ip", "client_port", - "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", "imei", "imsi", "apn", "phone_number", - "packet_capture_file", "in_src_mac", "out_src_mac", "server_ip", "server_port", - "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", "in_dest_mac", "out_dest_mac", - "app_path", - "app_debug_info", "app", - "app_content", - "protocol_path", + "app_category", + "decoded_path", "sent_pkts", "received_pkts", "sent_bytes", @@ -237,39 +266,25 @@ "tcp_c2s_rtx_bytes", "tcp_s2c_rtx_bytes", "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", "http_url", "http_host", - "http_request_line", - "http_response_line", - "http_proxy_flag", - "http_sequence", - "http_cookie", - "http_referer", "http_user_agent", "http_request_content_length", - "http_request_content_type", "http_response_content_length", - "http_response_content_type", - "http_set_cookie", - "http_version", "http_status_code", "http_response_latency_ms", "http_session_duration_ms", - "http_action_file_size", - "mail_protocol_type", "mail_account", "mail_from_cmd", "mail_to_cmd", "mail_from", - "mail_password", "mail_to", "mail_cc", "mail_bcc", "mail_subject", - "mail_subject_charset", "mail_attachment_name", - "mail_attachment_name_charset", - "mail_eml_file", "dns_message_id", "dns_qr", "dns_opcode", @@ -279,18 +294,9 @@ "dns_rcode", "dns_qtype", "dns_qclass", - "dns_qdcount", - "dns_ancount", - "dns_nscount", - "dns_arcount", "dns_qname", - "dns_qtype", - "dns_qclass", "dns_cname", - "dns_sub", - "dns_rr", "dns_response_latency_ms", - "ssl_version", "ssl_sni", "ssl_san", "ssl_cn", @@ -299,72 +305,20 @@ "ssl_ja3s_hash", "ssl_cert_issuer", "ssl_cert_subject", - "ssl_esni_flag", - "ssl_ech_flag", - "dtls_cookie", - "dtls_version", "dtls_sni", "dtls_san", "dtls_cn", "dtls_handshake_latency_ms", - "dtls_ja3_fingerprint", "dtls_ja3_hash", "dtls_cert_issuer", "dtls_cert_subject", "quic_sni", - "quic_version", "quic_user_agent", "ftp_account", "ftp_url", - "ftp_link_type", "sip_call_id", - "sip_originator_description", - "sip_responder_description", - "sip_user_agent", "sip_server", - "sip_originator_sdp_connect_ip", - "sip_originator_sdp_media_port", - "sip_originator_sdp_media_type", - "sip_originator_sdp_content", - "sip_responder_sdp_connect_ip", - "sip_responder_sdp_media_port", - "sip_responder_sdp_media_type", - "sip_responder_sdp_content", - "sip_duration_s", - "sip_bye", - "rtp_payload_type_c2s", - "rtp_payload_type_s2c", - "rtp_originator_dir", - "ssh_version", - "ssh_auth_success", - "ssh_client_version", - "ssh_server_version", - "ssh_cipher_alg", - "ssh_mac_alg", - "ssh_compression_alg", - "ssh_kex_alg", - "ssh_host_key_alg", - "ssh_host_key", - "ssh_hassh", - "stratum_cryptocurrency", - "stratum_mining_pools", - "stratum_mining_program", - "stratum_mining_subscribe", - "rdp_cookie", - "rdp_security_protocol", - "rdp_client_channels", - "rdp_keyboard_layout", - "rdp_client_version", - "rdp_client_name", - "rdp_client_product_id", - "rdp_desktop_width", - "rdp_desktop_height", - "rdp_requested_color_depth", - "rdp_certificate_type", - "rdp_certificate_count", - "rdp_certificate_permanent", - "rdp_encryption_level", - "rdp_encryption_method" + "ssh_hassh" ], "filters": [ "session_id", @@ -375,19 +329,30 @@ "processing_time", "ingestion_time", "device_id", + "out_link_id", + "in_link_id", "data_center", "sled_ip", "device_group", "address_type", "decoded_as", + "direction", "vsys_id", "flags", "flags_identify_info", + "c2s_ttl", + "s2c_ttl", "monitor_rule_list", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", "client_ip", "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", "imei", @@ -400,19 +365,27 @@ "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", "in_dest_mac", "out_dest_mac", - "app_path", + "app_transition", "app_debug_info", "app", + "app_category", "app_content", + "app_extra_info", "dup_traffic_flag", + "tunnel_id_list", "tunnel_endpoint_a_desc", "tunnel_endpoint_b_desc", - "protocol_path", + "ip_protocol", + "decoded_path", "fqdn_category_list", "sent_pkts", "received_pkts", @@ -462,6 +435,7 @@ "mail_attachment_name", "mail_attachment_name_charset", "mail_eml_file", + "mail_starttls_flag", "dns_message_id", "dns_qr", "dns_opcode", @@ -476,8 +450,6 @@ "dns_nscount", "dns_arcount", "dns_qname", - "dns_qtype", - "dns_qclass", "dns_cname", "dns_sub", "dns_rr", @@ -556,13 +528,175 @@ "rdp_certificate_count", "rdp_certificate_permanent", "rdp_encryption_level", - "rdp_encryption_method" + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "internal_port_list", + "external_port_list", + "internal_asn_list", + "external_asn_list", + "monitor_rule_id" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" }, "details": { - "$ref": "public_schema_info.json#/details" + "general": { + "$ref": "public_schema_info.json#/details/general" + }, + "treatment": { + "$ref": "public_schema_info.json#/details/treatment" + }, + "source": { + "$ref": "public_schema_info.json#/details/source" + }, + "destination": { + "$ref": "public_schema_info.json#/details/destination" + }, + "application": { + "$ref": "public_schema_info.json#/details/application" + }, + "protocol": [ + "ip_protocol", + "decoded_path", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "ftp_account", + "ftp_url", + "ftp_link_type", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "mail_starttls_flag", + "quic_version", + "quic_sni", + "quic_user_agent", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe" + ], + "transmission": { + "$ref": "public_schema_info.json#/details/transmission" + }, + "other": { + "$ref": "public_schema_info.json#/details/other" + } } }, "decoded_as": { @@ -596,6 +730,9 @@ "RTP": { "$ref": "public_schema_info.json#/decoded_as/RTP" }, + "RDP": { + "$ref": "public_schema_info.json#/decoded_as/RDP" + }, "SSH": { "$ref": "public_schema_info.json#/decoded_as/SSH" }, @@ -617,7 +754,7 @@ "recv_time", "log_id", "flags_identify_info", - "tunnels", + "encapsulation", "app_debug_info", "app_content", "packet_capture_file", @@ -628,7 +765,195 @@ "$ref": "public_schema_info.json#/tunnel_type" }, "measurements": { - "aggregates": { + "aggregated_metric_unit": { + "SUM": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes" + }, + "RATE": { + "sent_pkts": "pps", + "received_pkts": "pps", + "sent_bytes": "Bps", + "received_bytes": "Bps", + "tcp_c2s_lost_bytes": "Bps", + "tcp_s2c_lost_bytes": "Bps", + "tcp_c2s_o3_pkts": "pps", + "tcp_s2c_o3_pkts": "pps", + "tcp_c2s_rtx_pkts": "pps", + "tcp_s2c_rtx_pkts": "pps", + "tcp_c2s_rtx_bytes": "Bps", + "tcp_s2c_rtx_bytes": "Bps", + "http_request_content_length": "Bps", + "http_response_content_length": "Bps" + }, + "BITRATE": { + "sent_bytes": "bps", + "received_bytes": "bps", + "tcp_c2s_lost_bytes": "bps", + "tcp_s2c_lost_bytes": "bps", + "tcp_c2s_rtx_bytes": "bps", + "tcp_s2c_rtx_bytes": "bps", + "http_request_content_length": "bps", + "http_response_content_length": "bps" + }, + "MAX": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MIN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "AVG": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MEDIAN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P95_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P99_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + } + }, + "field_discovery_metric": { "sessions": [ { "fn": "count", @@ -666,7 +991,108 @@ } ] } - } + }, + "expression_fields": [ + { + "name": "internal_ip_list", + "label": "Internal IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_ip,''), if(bitAnd(flags, 16)=16, server_ip,''))" + } + }, + { + "name": "external_ip_list", + "label": "External IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_ip,''), if(bitAnd(flags, 16)!=16, server_ip,''))" + } + }, + { + "name": "internal_port_list", + "label": "Internal Port List", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_port, null), if(bitAnd(flags, 16)=16, server_port, null))" + } + }, + { + "name": "external_port_list", + "label": "External Port List", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_port, null), if(bitAnd(flags, 16)!=16, server_port, null))" + } + }, + { + "name": "internal_asn_list", + "label": "Internal ASN List", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_asn, null), if(bitAnd(flags, 16)=16, server_asn, null))" + } + }, + { + "name": "external_asn_list", + "label": "External ASN List", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_asn, null), if(bitAnd(flags, 16)!=16, server_asn, null))" + } + }, + { + "name": "monitor_rule_id", + "label": "Monitor Rule ID", + "type": "long", + "doc": { + "allow_query": "false", + "expression": "arrayJoin(IF(empty(monitor_rule_list), arrayPushFront(monitor_rule_list, null),monitor_rule_list))" + } + } + ] }, "fields": [ { @@ -688,11 +1114,9 @@ "name": "log_id", "label": "Log ID", "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], + "constraints": { + "aggregation_functions": "COUNT" + }, "visibility": "enabled", "ttl": null }, @@ -703,7 +1127,7 @@ "label": "Decoded AS", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -781,11 +1205,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -802,11 +1221,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -817,10 +1231,11 @@ }, { "name": "duration_ms", - "label": "Duration (ms)", + "label": "Duration(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -829,10 +1244,11 @@ }, { "name": "tcp_handshake_latency_ms", - "label": "TCP Handshake Latency (ms)", + "label": "TCP Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -846,11 +1262,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "current_timestamp" - } - ], "visibility": "enabled", "ttl": null }, @@ -866,15 +1277,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "ingestion_time" - }, - { - "function": "get_value", - "appendTo": "recv_time" - } - ], "visibility": "enabled", "ttl": null }, @@ -930,18 +1332,6 @@ "label": "Device Tag", "doc": { "visibility": "hidden", - "format": [ - { - "function": "flattenSpec", - "appendTo": "data_center", - "param": "$.tags[?(@.tag=='data_center')][0].value" - }, - { - "function": "flattenSpec", - "appendTo": "device_group", - "param": "$.tags[?(@.tag=='device_group')][0].value" - } - ], "ttl": null }, "type": "string" @@ -951,7 +1341,7 @@ "label": "Data Center", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -968,7 +1358,7 @@ "label": "Device Group", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -985,6 +1375,7 @@ "label": "Sled IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, "visibility": "enabled", @@ -999,22 +1390,30 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/address_type/data" + }, "visibility": "enabled", "ttl": null }, "type": "int" }, { + "name": "direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/direction/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { "name": "vsys_id", "label": "Vsys ID", "doc": { @@ -1063,6 +1462,30 @@ "type": "string" }, { + "name": "c2s_ttl", + "label": "Client-to-Server TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { + "name": "s2c_ttl", + "label": "Server-to-Client TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { "name": "security_rule_list", "label": "Security Rule List", "doc": { @@ -1084,18 +1507,11 @@ "doc": { "visibility": "hidden", "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/security_action/data" }, - "data": [ - { - "code": "Deny", - "value": "Deny" - }, - { - "code": "Allow", - "value": "Allow" - } - ], "ttl": null }, "type": "string" @@ -1133,8 +1549,24 @@ } }, { + "name": "statistics_rule_list", + "label": "Statistics Rule List", + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "hidden", + "ttl": null + }, + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + } + }, + { "name": "sc_rsp_raw", - "label": "Service Chaining Rendered Service Path (Raw)", + "label": "Service Chaining Rendered Service Path(Raw)", "doc": { "constraints": { "operator_functions": "has,notEmpty,empty" @@ -1150,7 +1582,7 @@ }, { "name": "sc_rsp_decrypted", - "label": "Service Chaining Rendered Service Path (Decrypted)", + "label": "Service Chaining Rendered Service Path(Decrypted)", "doc": { "constraints": { "operator_functions": "has,notEmpty,empty" @@ -1202,7 +1634,7 @@ "doc": { "visibility": "hidden", "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -1222,20 +1654,12 @@ "name": "proxy_pinning_status", "label": "Proxy Pinning Status", "doc": { - "data": [ - { - "code": "0", - "value": "not pinning" - }, - { - "code": "1", - "value": "pinning" - }, - { - "code": "2", - "value": "maybe pinning" - } - ], + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/proxy_pinning_status/data" + }, "visibility": "hidden", "ttl": null }, @@ -1245,20 +1669,12 @@ "name": "proxy_intercept_status", "label": "Proxy Intercept Status", "doc": { - "data": [ - { - "code": "0", - "value": "passthrough" - }, - { - "code": "1", - "value": "intercept" - }, - { - "code": "2", - "value": "shutdown" - } - ], + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/proxy_intercept_status/data" + }, "visibility": "hidden", "ttl": null }, @@ -1275,7 +1691,7 @@ }, { "name": "proxy_server_side_latency_ms", - "label": "Proxy Server Side Latency", + "label": "Proxy Server-Side Latency(ms)", "doc": { "visibility": "hidden", "ttl": null @@ -1284,7 +1700,7 @@ }, { "name": "proxy_client_side_latency_ms", - "label": "Proxy Client Side Latency", + "label": "Proxy Client-Side Latency(ms)", "doc": { "visibility": "hidden", "ttl": null @@ -1293,7 +1709,7 @@ }, { "name": "proxy_client_side_version", - "label": "Proxy Client Side Version", + "label": "Proxy Client-Side Version", "doc": { "visibility": "hidden", "ttl": null @@ -1302,7 +1718,7 @@ }, { "name": "proxy_server_side_version", - "label": "Proxy Server Side Version", + "label": "Proxy Server-Side Version", "doc": { "visibility": "hidden", "ttl": null @@ -1311,7 +1727,7 @@ }, { "name": "proxy_cert_verify", - "label": "Proxy Cert Verify", + "label": "Proxy Certificate Verify", "doc": { "visibility": "hidden", "ttl": null @@ -1328,26 +1744,26 @@ "type": "string" }, { - "name": "security_mirrored_pkts", - "label": "Security Mirrored Packets", + "name": "monitor_mirrored_pkts", + "label": "Monitor Mirrored Packets", "type": "int", "doc": { "constraints": { "type": "decimal" }, - "visibility": "hidden", + "visibility": "enabled", "ttl": null } }, { - "name": "security_mirrored_bytes", - "label": "Security Mirrored Bytes", + "name": "monitor_mirrored_bytes", + "label": "Monitor Mirrored Bytes", "type": "int", "doc": { "constraints": { "type": "bytes" }, - "visibility": "hidden", + "visibility": "enabled", "ttl": null } }, @@ -1356,18 +1772,9 @@ "label": "Client IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "client_asn" - }, - { - "function": "radius_match", - "appendTo": "subscriber_id" - } - ], "visibility": "enabled", "ttl": null }, @@ -1377,6 +1784,10 @@ "name": "client_port", "label": "Client Port", "doc": { + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1393,7 +1804,52 @@ }, { "name": "client_geolocation", - "label": "Client Geolocation", + "label": "Client Geolocation(Deprecated)", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_country", + "label": "Client Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_super_administrative_area", + "label": "Client Super Administrative Area", + "doc": { + "constraints": { + "type": "super_administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_administrative_area", + "label": "Client Administrative Area", + "doc": { + "constraints": { + "type": "administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_sub_administrative_area", + "label": "Client SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1404,6 +1860,9 @@ "name": "client_asn", "label": "Client ASN", "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1459,14 +1918,9 @@ "label": "Server IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "server_asn" - } - ], "visibility": "enabled", "ttl": null }, @@ -1476,6 +1930,10 @@ "name": "server_port", "label": "Server Port", "doc": { + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1492,7 +1950,52 @@ }, { "name": "server_geolocation", - "label": "Server Geolocation", + "label": "Server Geolocation(Deprecated)", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_country", + "label": "Server Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_super_administrative_area", + "label": "Server Super Administrative Area", + "doc": { + "constraints": { + "type": "super_administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_administrative_area", + "label": "Server Administrative Area", + "doc": { + "constraints": { + "type": "administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_sub_administrative_area", + "label": "Server SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1503,6 +2006,9 @@ "name": "server_asn", "label": "Server ASN", "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1512,6 +2018,9 @@ "name": "server_fqdn", "label": "Server FQDN", "doc": { + "constraints": { + "type": "fqdn" + }, "visibility": "enabled", "ttl": null }, @@ -1521,14 +2030,17 @@ "name": "server_domain", "label": "Server Domain", "doc": { + "constraints": { + "type": "domain" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "app_path", - "label": "Application Path", + "name": "app_transition", + "label": "Application Transition", "type": "string", "doc": { "visibility": "enabled", @@ -1539,6 +2051,18 @@ "name": "app", "label": "Application", "doc": { + "constraints": { + "type": "app" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "app_category", + "label": "Application Category", + "doc": { "visibility": "enabled", "ttl": null }, @@ -1548,7 +2072,7 @@ "name": "app_debug_info", "label": "Application Debug Info", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1563,8 +2087,32 @@ "type": "string" }, { - "name": "protocol_path", - "label": "Protocol Path", + "name": "app_extra_info", + "label": "Application Extra Info", + "doc": { + "visibility": "hidden", + "ttl": null + }, + "type": "string" + }, + { + "name": "ip_protocol", + "label": "IP Protocol", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/ip_protocol/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "decoded_path", + "label": "Decoded Path", "doc": { "visibility": "enabled", "ttl": null @@ -1579,16 +2127,16 @@ "operator_functions": "has,notEmpty,empty" }, "dict_location": { - "path": "/v1/category/dict", - "key": "categoryId", - "value": "categoryName" + "path": "/v1/policy/object?type=fqdn_category", + "key": "category_id", + "value": "category_name" }, "visibility": "enabled", "ttl": null }, "type": { "type": "array", - "items": "int", + "items": "long", "logicalType": "array" } }, @@ -1642,7 +2190,7 @@ }, { "name": "tcp_c2s_ip_fragments", - "label": "TCP C2S IP Fragments ", + "label": "Client-to-Server IP Fragments", "doc": { "constraints": { "type": "decimal" @@ -1654,7 +2202,7 @@ }, { "name": "tcp_s2c_ip_fragments", - "label": "TCP S2C IP Fragments", + "label": "Server-to-Client IP Fragments", "doc": { "constraints": { "type": "decimal" @@ -1666,7 +2214,7 @@ }, { "name": "tcp_c2s_lost_bytes", - "label": "TCP C2S Lost Bytes", + "label": "Client-to-Server Lost Bytes", "doc": { "constraints": { "type": "bytes" @@ -1678,7 +2226,7 @@ }, { "name": "tcp_s2c_lost_bytes", - "label": "TCP S2C Lost Bytes", + "label": "Server-to-Client Lost Bytes", "doc": { "constraints": { "type": "bytes" @@ -1690,7 +2238,7 @@ }, { "name": "tcp_c2s_o3_pkts", - "label": "TCP C2S out-of-order packets", + "label": "Client-to-Server Out-of-Order Packets", "doc": { "constraints": { "type": "decimal" @@ -1702,7 +2250,7 @@ }, { "name": "tcp_s2c_o3_pkts", - "label": "TCP S2C out-of-order packets", + "label": "Server-to-Client Out-of-Order Packets", "doc": { "constraints": { "type": "decimal" @@ -1714,7 +2262,7 @@ }, { "name": "tcp_c2s_rtx_pkts", - "label": "TCP C2S Retransmission Packets", + "label": "Client-to-Server Retransmission Packets", "doc": { "constraints": { "type": "decimal" @@ -1726,7 +2274,7 @@ }, { "name": "tcp_s2c_rtx_pkts", - "label": "TCP S2C Retransmission Packets", + "label": "Server-to-Client Retransmission Packets", "doc": { "constraints": { "type": "decimal" @@ -1738,7 +2286,7 @@ }, { "name": "tcp_c2s_rtx_bytes", - "label": "TCP C2S Retransmission Bytes ", + "label": "Client-to-Server Retransmission Bytes", "doc": { "constraints": { "type": "bytes" @@ -1750,7 +2298,7 @@ }, { "name": "tcp_s2c_rtx_bytes", - "label": "TCP S2C Retransmission Bytes", + "label": "Server-to-Client Retransmission Bytes", "doc": { "constraints": { "type": "bytes" @@ -1762,10 +2310,11 @@ }, { "name": "tcp_rtt_ms", - "label": "Round-trip Time", + "label": "Round-trip Time(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -1774,7 +2323,7 @@ }, { "name": "tcp_client_isn", - "label": "TCP Client ISN", + "label": "Client ISN", "doc": { "visibility": "enabled", "ttl": null @@ -1783,7 +2332,7 @@ }, { "name": "tcp_server_isn", - "label": "TCP Server ISN", + "label": "Server ISN", "doc": { "visibility": "enabled", "ttl": null @@ -1840,18 +2389,11 @@ "type": "string" }, { - "name": "tunnels", - "label": "Tunnels", + "name": "encapsulation", + "label": "Encapsulation", "doc": { "visibility": "enabled", - "ttl": null, - "format": [ - { - "function": "gtpc_match", - "appendTo": "imsi,imei,phone_number", - "param": "$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_a2b_teid,$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_b2a_teid" - } - ] + "ttl": null }, "type": "string" }, @@ -1871,8 +2413,21 @@ "type": "int" }, { + "name": "tunnel_id_list", + "label": "Tunnel ID List", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + } + }, + { "name": "tunnel_endpoint_a_desc", - "label": "Tunnel Endpoint A Description", + "label": "Tunnel Endpoint A Description(Deprecated)", "doc": { "visibility": "enabled", "ttl": null @@ -1881,7 +2436,7 @@ }, { "name": "tunnel_endpoint_b_desc", - "label": "Tunnel Endpoint B Description", + "label": "Tunnel Endpoint B Description(Deprecated)", "doc": { "visibility": "enabled", "ttl": null @@ -1892,6 +2447,9 @@ "name": "http_url", "label": "HTTP.URL", "doc": { + "constraints": { + "type": "url" + }, "visibility": "enabled", "ttl": null }, @@ -1926,16 +2484,16 @@ }, { "name": "http_request_content_length", - "label": "HTTP.Request Content Length", + "label": "HTTP.Request Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_request_content_type", - "label": "HTTP.Request Content Type", + "label": "HTTP.Request Content-Type", "doc": { "visibility": "enabled", "ttl": null @@ -1944,12 +2502,12 @@ }, { "name": "http_response_content_length", - "label": "HTTP.Response Content Length", + "label": "HTTP.Response Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_response_content_type", @@ -1964,11 +2522,6 @@ "name": "http_request_body", "label": "HTTP.Request Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -1982,11 +2535,6 @@ "name": "http_response_body", "label": "HTTP.Response Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -2000,7 +2548,7 @@ "name": "http_proxy_flag", "label": "HTTP.Proxy Flag", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "int" @@ -2034,7 +2582,7 @@ }, { "name": "http_user_agent", - "label": "HTTP.User Agent", + "label": "HTTP.User-Agent", "doc": { "visibility": "enabled", "ttl": null @@ -2043,7 +2591,7 @@ }, { "name": "http_set_cookie", - "label": "HTTP.Set Cookie", + "label": "HTTP.Set-Cookie", "doc": { "visibility": "enabled", "ttl": null @@ -2064,21 +2612,26 @@ "label": "HTTP.Status Code", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT", + "type": "http_code" + }, "ttl": null }, "type": "int" }, { "name": "http_response_latency_ms", - "label": "HTTP.Response Latency (ms)", + "label": "HTTP.Response Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "http_action_file_size", @@ -2090,23 +2643,24 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "http_session_duration_ms", - "label": "HTTP.Session Duration (ms)", + "label": "HTTP.Session Duration(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "mail_protocol_type", - "label": "Mail.Protocol Type", + "label": "MAIL.Protocol Type", "doc": { "visibility": "enabled", "ttl": null @@ -2115,8 +2669,11 @@ }, { "name": "mail_account", - "label": "Mail.Account", + "label": "MAIL.Account", "doc": { + "constraints": { + "type": "email" + }, "visibility": "enabled", "ttl": null }, @@ -2124,7 +2681,7 @@ }, { "name": "mail_from_cmd", - "label": "Mail.From CMD", + "label": "MAIL.From CMD", "doc": { "visibility": "enabled", "ttl": null @@ -2133,7 +2690,7 @@ }, { "name": "mail_to_cmd", - "label": "Mail.To CMD", + "label": "MAIL.To CMD", "doc": { "visibility": "enabled", "ttl": null @@ -2142,7 +2699,7 @@ }, { "name": "mail_from", - "label": "Mail.From", + "label": "MAIL.From", "doc": { "constraints": { "type": "email" @@ -2154,7 +2711,7 @@ }, { "name": "mail_password", - "label": "Mail.Password", + "label": "MAIL.Password", "doc": { "visibility": "enabled", "ttl": null @@ -2163,7 +2720,7 @@ }, { "name": "mail_to", - "label": "Mail.To", + "label": "MAIL.To", "doc": { "constraints": { "type": "email" @@ -2175,7 +2732,7 @@ }, { "name": "mail_cc", - "label": "Mail.CC", + "label": "MAIL.CC", "doc": { "visibility": "enabled", "ttl": null @@ -2184,7 +2741,7 @@ }, { "name": "mail_bcc", - "label": "Mail.BCC", + "label": "MAIL.BCC", "doc": { "visibility": "enabled", "ttl": null @@ -2193,14 +2750,8 @@ }, { "name": "mail_subject", - "label": "Mail.Subject", + "label": "MAIL.Subject", "doc": { - "format": [ - { - "function": "decode_of_base64", - "param": "$.mail_subject_charset" - } - ], "visibility": "enabled", "ttl": null }, @@ -2208,7 +2759,7 @@ }, { "name": "mail_subject_charset", - "label": "Mail.Subject Charset", + "label": "MAIL.Subject Charset", "doc": { "visibility": "enabled", "ttl": null @@ -2217,14 +2768,8 @@ }, { "name": "mail_attachment_name", - "label": "Mail.Attachment", + "label": "MAIL.Attachment Name", "doc": { - "format": [ - { - "function": "decode_of_base64", - "param": "$.mail_attachment_name_charset" - } - ], "visibility": "enabled", "ttl": null }, @@ -2232,7 +2777,7 @@ }, { "name": "mail_attachment_name_charset", - "label": "Mail.Attachment Charset", + "label": "MAIL.Attachment Name Charset", "doc": { "visibility": "enabled", "ttl": null @@ -2241,13 +2786,8 @@ }, { "name": "mail_eml_file", - "label": "Mail.EML File", + "label": "MAIL.EML File", "doc": { - "format": [ - { - "function": "combine" - } - ], "constraints": { "type": "file" }, @@ -2257,6 +2797,21 @@ "type": "string" }, { + "name": "mail_starttls_flag", + "label": "MAIL.STARTTLS Flag", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/mail_starttls_flag/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { "name": "dns_message_id", "label": "DNS.Message ID", "doc": { @@ -2272,16 +2827,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "QUERY" - }, - { - "code": "1", - "value": "RESPONSE" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_qr/data" + }, "visibility": "enabled", "ttl": null }, @@ -2292,26 +2840,12 @@ "label": "DNS.OPCODE", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": { + "$ref": "public_schema_info.json#/fields/dns_opcode/data" }, - "data": [ - { - "code": "0", - "value": "QUERY" - }, - { - "code": "1", - "value": "IQUERY" - }, - { - "code": "2", - "value": "STATUS" - }, - { - "code": "5", - "value": "UPDATE" - } - ], "visibility": "enabled", "ttl": null }, @@ -2322,6 +2856,9 @@ "label": "DNS.AA", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -2357,77 +2894,14 @@ "name": "dns_rcode", "label": "DNS.RCODE", "doc": { - "data": [ - { - "code": 0, - "value": "NoError" - }, - { - "code": 1, - "value": "FormErr" - }, - { - "code": 2, - "value": "ServFail" - }, - { - "code": 3, - "value": "NXDomain" - }, - { - "code": 4, - "value": "NotImp" - }, - { - "code": 5, - "value": "Refused" - }, - { - "code": 6, - "value": "YXDomain" - }, - { - "code": 7, - "value": "YXRRSet" - }, - { - "code": 8, - "value": "NXRRSet" - }, - { - "code": 9, - "value": "NotAuth" - }, - { - "code": 10, - "value": "NotZone" - }, - { - "code": 16, - "value": "BADSIG" - }, - { - "code": 17, - "value": "BADKEY" - }, - { - "code": 18, - "value": "BADTIME" - }, - { - "code": 19, - "value": "BADMODE" - }, - { - "code": 20, - "value": "BADNAME" - }, - { - "code": 21, - "value": "BADALG" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_rcode/data" + }, "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -2494,330 +2968,12 @@ "label": "DNS.QTYPE", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": { + "$ref": "public_schema_info.json#/fields/dns_qtype/data" }, - "data": [ - { - "code": "1", - "value": "A" - }, - { - "code": "2", - "value": "NS" - }, - { - "code": "3", - "value": "MD" - }, - { - "code": "4", - "value": "MF" - }, - { - "code": "5", - "value": "CNAME" - }, - { - "code": "6", - "value": "SOA" - }, - { - "code": "7", - "value": "MB" - }, - { - "code": "8", - "value": "MG" - }, - { - "code": "9", - "value": "MR" - }, - { - "code": "10", - "value": "NULL" - }, - { - "code": "11", - "value": "WKS" - }, - { - "code": "12", - "value": "PTR" - }, - { - "code": "13", - "value": "HINFO" - }, - { - "code": "14", - "value": "MINFO" - }, - { - "code": "15", - "value": "MX" - }, - { - "code": "16", - "value": "TXT" - }, - { - "code": "17", - "value": "RP" - }, - { - "code": "18", - "value": "AFSDB" - }, - { - "code": "19", - "value": "X25" - }, - { - "code": "20", - "value": "ISDN" - }, - { - "code": "21", - "value": "RT" - }, - { - "code": "22", - "value": "NSAP" - }, - { - "code": "23", - "value": "NSAP" - }, - { - "code": "24", - "value": "SIG" - }, - { - "code": "25", - "value": "KEY" - }, - { - "code": "26", - "value": "PX" - }, - { - "code": "27", - "value": "GPOS" - }, - { - "code": "28", - "value": "AAAA" - }, - { - "code": "29", - "value": "LOC" - }, - { - "code": "30", - "value": "EID" - }, - { - "code": "31", - "value": "NIMLOC" - }, - { - "code": "32", - "value": "NB" - }, - { - "code": "33", - "value": "SRV" - }, - { - "code": "34", - "value": "ATMA" - }, - { - "code": "35", - "value": "NAPTR" - }, - { - "code": "36", - "value": "KX" - }, - { - "code": "37", - "value": "CERT" - }, - { - "code": "38", - "value": "A6" - }, - { - "code": "39", - "value": "DNAME" - }, - { - "code": "40", - "value": "SINK" - }, - { - "code": "41", - "value": "OPT" - }, - { - "code": "42", - "value": "APL" - }, - { - "code": "43", - "value": "DS" - }, - { - "code": "44", - "value": "SSHFP" - }, - { - "code": "45", - "value": "IPSECKEY" - }, - { - "code": "46", - "value": "RRSIG" - }, - { - "code": "47", - "value": "NSEC" - }, - { - "code": "48", - "value": "DNSKEY" - }, - { - "code": "49", - "value": "DHCID" - }, - { - "code": "50", - "value": "NSEC3" - }, - { - "code": "51", - "value": "NSEC3PARAM" - }, - { - "code": "52", - "value": "TLSA" - }, - { - "code": "53", - "value": "SMIMEA" - }, - { - "code": "55", - "value": "HIP" - }, - { - "code": "59", - "value": "CDS" - }, - { - "code": "60", - "value": "CDNSKEY" - }, - { - "code": "61", - "value": "OPENPGPKEY" - }, - { - "code": "62", - "value": "CSYNC" - }, - { - "code": "63", - "value": "ZONEMD" - }, - { - "code": "64", - "value": "SVCB" - }, - { - "code": "65", - "value": "HTTPS" - }, - { - "code": "99", - "value": "SPF" - }, - { - "code": "100", - "value": "UINFO" - }, - { - "code": "101", - "value": "UID" - }, - { - "code": "102", - "value": "GID" - }, - { - "code": "103", - "value": "UNSPEC" - }, - { - "code": "108", - "value": "EUI48" - }, - { - "code": "109", - "value": "EUI64" - }, - { - "code": "249", - "value": "TKEY" - }, - { - "code": "250", - "value": "TSIG" - }, - { - "code": "251", - "value": "IXFR" - }, - { - "code": "252", - "value": "AXFR" - }, - { - "code": "253", - "value": "MAILB" - }, - { - "code": "254", - "value": "MAILA" - }, - { - "code": "255", - "value": "*" - }, - { - "code": "256", - "value": "URI" - }, - { - "code": "257", - "value": "CAA" - }, - { - "code": "32768", - "value": "TA" - }, - { - "code": "32769", - "value": "DLV" - }, - { - "code": "65521", - "value": "INTEGRITY" - } - ], "visibility": "enabled", "ttl": null }, @@ -2828,6 +2984,10 @@ "label": "DNS.QCLASS", "doc": { "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -2848,16 +3008,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "1", - "value": "DNS" - }, - { - "code": "2", - "value": "DNSSEC" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_sub/data" + }, "visibility": "enabled", "ttl": null }, @@ -2874,12 +3027,13 @@ }, { "name": "dns_response_latency_ms", - "label": "DNS.Response Latency (ms)", + "label": "DNS.Response Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "int" @@ -2922,10 +3076,11 @@ }, { "name": "ssl_handshake_latency_ms", - "label": "SSL.Handshake Latency (ms)", + "label": "SSL.Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -2934,7 +3089,7 @@ }, { "name": "ssl_ja3_hash", - "label": "SSL.JA3 hash", + "label": "SSL.JA3 Hash", "doc": { "visibility": "enabled", "ttl": null @@ -2943,7 +3098,7 @@ }, { "name": "ssl_ja3s_hash", - "label": "SSL.JA3S hash", + "label": "SSL.JA3S Hash", "doc": { "visibility": "enabled", "ttl": null @@ -2978,6 +3133,12 @@ "name": "ssl_esni_flag", "label": "SSL.ESNI Flag", "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/ssl_esni_flag/data" + }, "visibility": "enabled", "ttl": null }, @@ -2987,6 +3148,12 @@ "name": "ssl_ech_flag", "label": " SSL.ECH Flag", "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/ssl_ech_flag/data" + }, "visibility": "enabled", "ttl": null }, @@ -2996,7 +3163,7 @@ "name": "dtls_cookie", "label": "DTLS.Cookie", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" @@ -3005,7 +3172,7 @@ "name": "dtls_version", "label": "DTLS.Version", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" @@ -3023,7 +3190,7 @@ "name": "dtls_san", "label": "DTLS.SAN", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" @@ -3032,62 +3199,63 @@ "name": "dtls_cn", "label": "DTLS.CN", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_handshake_latency_ms", - "label": "DTLS.Handshake Latency (ms)", + "label": "DTLS.Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "int" }, { "name": "dtls_ja3_fingerprint", - "label": "DTLS.JA3", + "label": "DTLS.JA3 Fingerprint", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_ja3_hash", - "label": "DTLS.JA3 hash", + "label": "DTLS.JA3 Hash", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_cert_issuer", - "label": "DTLS.Issuer", + "label": "DTLS.Certificate Issuer", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_cert_subject", - "label": "DTLS.Subject", + "label": "DTLS.Certificate Subject", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "quic_version", - "label": "Quic.Version", + "label": "QUIC.Version", "doc": { "visibility": "enabled", "ttl": null @@ -3096,7 +3264,7 @@ }, { "name": "quic_sni", - "label": "Quic.SNI", + "label": "QUIC.SNI", "doc": { "visibility": "enabled", "ttl": null @@ -3105,7 +3273,7 @@ }, { "name": "quic_user_agent", - "label": "Quic.User Agent", + "label": "QUIC.User-Agent", "doc": { "visibility": "enabled", "ttl": null @@ -3188,6 +3356,9 @@ "name": "sip_originator_sdp_connect_ip", "label": "SIP.Originator IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -3197,6 +3368,9 @@ "name": "sip_originator_sdp_media_port", "label": "SIP.Originator Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -3224,6 +3398,9 @@ "name": "sip_responder_sdp_connect_ip", "label": "SIP.Responder IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -3233,6 +3410,9 @@ "name": "sip_responder_sdp_media_port", "label": "SIP.Responder Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -3258,7 +3438,7 @@ }, { "name": "sip_duration_s", - "label": "SIP.Duration (s)", + "label": "SIP.Duration(s)", "doc": { "constraints": { "type": "decimal" @@ -3279,121 +3459,14 @@ }, { "name": "rtp_payload_type_c2s", - "label": "RTP.Payload Type (c2s)", + "label": "RTP.Payload Type(Client-to-Server)", "doc": { "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "PCMU" - }, - { - "code": "1", - "value": "1016" - }, - { - "code": "2", - "value": "G721" - }, - { - "code": "3", - "value": "GSM" - }, - { - "code": "4", - "value": "G723" - }, - { - "code": "5", - "value": "DVI4_8000" - }, - { - "code": "6", - "value": "DVI4_16000" - }, - { - "code": "7", - "value": "LPC" - }, - { - "code": "8", - "value": "PCMA" - }, - { - "code": "9", - "value": "G722" - }, - { - "code": "10", - "value": "L16_STEREO" - }, - { - "code": "11", - "value": "L16_MONO" - }, - { - "code": "12", - "value": "QCELP" - }, - { - "code": "13", - "value": "CN" - }, - { - "code": "14", - "value": "MPA" - }, - { - "code": "15", - "value": "G728" - }, - { - "code": "16", - "value": "DVI4_11025" - }, - { - "code": "17", - "value": "DVI4_22050" - }, - { - "code": "18", - "value": "G729" - }, - { - "code": "19", - "value": "CN_OLD" - }, - { - "code": "25", - "value": "CELB" - }, - { - "code": "26", - "value": "JPEG" - }, - { - "code": "28", - "value": "NV" - }, - { - "code": "31", - "value": "H261" - }, - { - "code": "32", - "value": "MPV" - }, - { - "code": "33", - "value": "MP2T" - }, - { - "code": "34", - "value": "H263" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_payload_type_c2s/data" + }, "visibility": "enabled", "ttl": null }, @@ -3401,121 +3474,14 @@ }, { "name": "rtp_payload_type_s2c", - "label": "RTP.Payload Type (s2c)", + "label": "RTP.Payload Type(Server-to-Client)", "doc": { "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "PCMU" - }, - { - "code": "1", - "value": "1016" - }, - { - "code": "2", - "value": "G721" - }, - { - "code": "3", - "value": "GSM" - }, - { - "code": "4", - "value": "G723" - }, - { - "code": "5", - "value": "DVI4_8000" - }, - { - "code": "6", - "value": "DVI4_16000" - }, - { - "code": "7", - "value": "LPC" - }, - { - "code": "8", - "value": "PCMA" - }, - { - "code": "9", - "value": "G722" - }, - { - "code": "10", - "value": "L16_STEREO" - }, - { - "code": "11", - "value": "L16_MONO" - }, - { - "code": "12", - "value": "QCELP" - }, - { - "code": "13", - "value": "CN" - }, - { - "code": "14", - "value": "MPA" - }, - { - "code": "15", - "value": "G728" - }, - { - "code": "16", - "value": "DVI4_11025" - }, - { - "code": "17", - "value": "DVI4_22050" - }, - { - "code": "18", - "value": "G729" - }, - { - "code": "19", - "value": "CN_OLD" - }, - { - "code": "25", - "value": "CELB" - }, - { - "code": "26", - "value": "JPEG" - }, - { - "code": "28", - "value": "NV" - }, - { - "code": "31", - "value": "H261" - }, - { - "code": "32", - "value": "MPV" - }, - { - "code": "33", - "value": "MP2T" - }, - { - "code": "34", - "value": "H263" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_payload_type_s2c/data" + }, "visibility": "enabled", "ttl": null }, @@ -3541,20 +3507,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "unknown" - }, - { - "code": "1", - "value": "c2s" - }, - { - "code": "2", - "value": "s2c" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_originator_dir/data" + }, "visibility": "enabled", "ttl": null }, @@ -3625,7 +3580,7 @@ }, { "name": "ssh_kex_alg", - "label": "SSH. Key Exchange Algorithm", + "label": "SSH.Key Exchange Algorithm", "doc": { "visibility": "enabled", "ttl": null @@ -3760,7 +3715,7 @@ }, { "name": "rdp_desktop_width", - "label": "RDP. Desktop Width", + "label": "RDP.Desktop Width", "doc": { "visibility": "enabled", "ttl": null diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/monitor_rule_hits.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/monitor_rule_hits.json new file mode 100644 index 00000000..c36d3f78 --- /dev/null +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/monitor_rule_hits.json @@ -0,0 +1,225 @@ +{ + "type": "record", + "name": "monitor_rule_hits", + "namespace": "druid", + "doc": { + "partition_key": "__time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "time": [ + "__time" + ], + "dimensions": [ + "rule_id", + "device_group", + "data_center" + ], + "metrics": [ + "hit_count", + "in_bytes", + "out_bytes", + "in_pkts", + "out_pkts" + ], + "filters": [ + "device_group", + "data_center", + "rule_id" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + } + }, + "measurements": { + "ingestion_period": "PT1S", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "hit_count": "sessions", + "in_bytes": "bytes", + "out_bytes": "bytes", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "hit_count": "sps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps" + } + } + } + }, + "fields": [ + { + "name": "__time", + "label": "Time", + "type": { + "type": "string", + "logicalType": "timestamp" + }, + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "enabled" + } + }, + { + "name": "vsys_id", + "label": "Vsys ID", + "doc": { + "visibility": "enabled" + }, + "type": "long", + "default": 1 + }, + { + "name": "device_id", + "label": "Device ID", + "doc": { + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "rule_id", + "label": "Rule", + "type": "long", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "visibility": "enabled" + } + }, + { + "name": "action", + "label": "Action", + "type": "long", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": [ + { + "code": 1, + "value": "Monitor" + }, + { + "code": 16, + "value": "Deny" + }, + { + "code": 96, + "value": "Allow" + }, + { + "code": 128, + "value": "Shunt" + } + ], + "visibility": "enabled" + } + }, + { + "name": "hit_count", + "label": "Hit Count", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "in_bytes", + "label": "Incoming Bytes", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "out_bytes", + "label": "Outgoing Bytes", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "out_pkts", + "label": "Outgoing Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/object_statistics.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/object_statistics.json index 075c6d41..c80d1058 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/object_statistics.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/object_statistics.json @@ -8,27 +8,21 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", "device_id", "device_group", "data_center", "template_id", "chart_id", "version", + "object_type", "object_id", "item_id" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "template_id", - "chart_id", - "version", - "object_id", - "item_id", "in_bytes", "out_bytes", "bytes", @@ -37,29 +31,48 @@ "sessions" ], "filters": [ - "vsys_id", "device_id", "device_group", "data_center", "template_id", "chart_id", "version", + "object_type", "object_id", - "item_id", - "in_bytes", - "out_bytes", - "bytes", - "new_in_sessions", - "new_out_sessions", - "sessions" + "item_id" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 15, - "ingestion_delay": 15 + "ingestion_period": "PT1S", + "compaction_period": "PT1M", + "skip_compaction_offset_from_latest": "P7D", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "in_bytes": "bytes", + "out_bytes": "bytes", + "bytes": "bytes", + "new_in_sessions": "sessions", + "new_out_sessions": "sessions", + "sessions": "sessions" + }, + "RATE": { + "in_bytes": "Bps", + "out_bytes": "Bps", + "bytes": "Bps", + "new_in_sessions": "sps", + "new_out_sessions": "sps", + "sessions": "sps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps", + "bytes": "bps" + } + } } }, "fields": [ @@ -83,7 +96,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -151,8 +164,73 @@ "type": "long" }, { + "name": "object_type", + "label": "Object Type", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": [ + { + "code": "ip", + "value": "IP" + }, + { + "code": "url", + "value": "URL" + }, + { + "code": "fqdn", + "value": "FQDN" + }, + { + "code": "account", + "value": "Account" + }, + { + "code": "subscriberid", + "value": "Subscriber ID" + }, + { + "code": "keywords", + "value": "Keywords" + }, + { + "code": "http_signature", + "value": "HTTP Signature" + }, + { + "code": "mobile_identity", + "value": "Mobile Identity" + }, + { + "code": "apn", + "value": "APN" + }, + { + "code": "flag", + "value": "Flag" + }, + { + "code": "application", + "value": "Application" + }, + { + "code": "tunnel", + "value": "Tunnel" + }, + { + "code": "fqdn_category", + "value": "FQDN Category" + } + ], + "visibility": "enabled" + }, + "type": "string" + }, + { "name": "object_id", - "label": "Object", + "label": "Object ID", "doc": { "visibility": "enabled" }, @@ -164,14 +242,15 @@ "doc": { "visibility": "enabled" }, - "type": "int" + "type": "long" }, { "name": "in_bytes", "label": "Incoming Bytes", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" }, "visibility": "enabled" }, @@ -182,7 +261,8 @@ "label": "Outgoing Bytes", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" }, "visibility": "enabled" }, @@ -193,7 +273,8 @@ "label": "Bytes", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" }, "visibility": "enabled" }, @@ -204,7 +285,8 @@ "label": "New Incoming Sessions", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -215,7 +297,8 @@ "label": "New Outgoing Sessions", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -226,7 +309,8 @@ "label": "Sessions", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/processes.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/processes_cluster.json index 75d74a98..dc9902ea 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/processes.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/processes_cluster.json @@ -1,7 +1,7 @@ { "namespace": "system", "type": "record", - "name": "processes", + "name": "processes_cluster", "fields": [ { "name": "query_id", diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/proxy_event.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/proxy_event.json index 322295b9..5b917323 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/proxy_event.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/proxy_event.json @@ -5,21 +5,30 @@ "doc": { "primary_key": "log_id", "partition_key": "recv_time", - "index_key": { - "$ref": "public_schema_info.json#/proxy_event/index_key" - }, + "index_key": [ + "vsys_id", + "proxy_action", + "decoded_as", + "data_center", + "device_group", + "recv_time" + ], "ttl": null, "default_ttl": 2592000, "functions": { "$ref": "public_schema_info.json#/functions" }, "schema_query": { - "dimensions": [ - "session_id", + "time": [ + "recv_time", "start_timestamp_ms", "end_timestamp_ms", "processing_time", "ingestion_time", + "insert_time" + ], + "dimensions": [ + "session_id", "device_id", "out_link_id", "in_link_id", @@ -28,41 +37,33 @@ "device_group", "address_type", "decoded_as", + "direction", "vsys_id", - "flags", - "flags_identify_info", "proxy_rule_list", "proxy_action", "client_ip", "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", - "imei", - "imsi", - "apn", - "phone_number", - "packet_capture_file", - "in_src_mac", - "out_src_mac", "server_ip", "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", - "in_dest_mac", - "out_dest_mac", - "app_path", - "app", - "app_debug_info", - "app_content", - "protocol_path", - "dup_traffic_flag", - "tunnel_endpoint_a_desc", - "tunnel_endpoint_b_desc", + "ip_protocol", + "tunnel_id_list", "http_url", "http_host", "http_request_line", @@ -80,12 +81,9 @@ "http_version", "http_status_code", "http_response_latency_ms", - "http_session_duration_ms", "http_action_file_size", "doh_url", "doh_host", - "doh_url", - "doh_host", "doh_request_line", "doh_response_line", "doh_cookie", @@ -112,180 +110,97 @@ "doh_qclass", "doh_cname", "doh_sub", - "doh_rr" + "doh_rr", + "internal_ip_list", + "external_ip_list", + "internal_port_list", + "external_port_list", + "internal_asn_list", + "external_asn_list", + "proxy_rule_id" ], "metrics": [ - "start_timestamp_ms", - "end_timestamp_ms", - "device_id", - "data_center", - "sled_ip", - "device_group", - "address_type", - "decoded_as", - "vsys_id", - "proxy_rule_list", - "proxy_action", + "log_id", "client_ip", "client_port", - "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", - "imei", - "imsi", - "apn", - "phone_number", - "packet_capture_file", - "in_src_mac", - "out_src_mac", "server_ip", "server_port", - "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", - "in_dest_mac", - "out_dest_mac", - "app_path", - "app_debug_info", - "app", - "app_content", - "protocol_path", - "sent_pkts", - "received_pkts", "sent_bytes", "received_bytes", - "duration_ms", - "tcp_handshake_latency_ms", - "tcp_c2s_ip_fragments", - "tcp_s2c_ip_fragments", - "tcp_c2s_lost_bytes", - "tcp_s2c_lost_bytes", - "tcp_c2s_o3_pkts", - "tcp_s2c_o3_pkts", - "tcp_c2s_rtx_pkts", - "tcp_s2c_rtx_pkts", - "tcp_c2s_rtx_bytes", - "tcp_s2c_rtx_bytes", "tcp_rtt_ms", "http_url", "http_host", - "http_request_line", - "http_response_line", - "http_proxy_flag", - "http_sequence", - "http_cookie", - "http_referer", "http_user_agent", "http_request_content_length", - "http_request_content_type", "http_response_content_length", - "http_response_content_type", - "http_set_cookie", - "http_version", "http_status_code", "http_response_latency_ms", - "http_session_duration_ms", - "http_action_file_size", - "doh_url", - "doh_host", "doh_url", "doh_host", - "doh_request_line", - "doh_response_line", - "doh_cookie", - "doh_referer", "doh_user_agent", "doh_content_length", - "doh_content_type", - "doh_set_cookie", - "doh_version", "doh_message_id", - "doh_qr", - "doh_opcode", - "doh_aa", - "doh_tc", - "doh_rd", - "doh_ra", - "doh_rcode", - "doh_qdcount", - "doh_ancount", - "doh_nscount", - "doh_arcount", - "doh_qname", - "doh_qtype", - "doh_qclass", - "doh_cname", - "doh_sub", - "doh_rr" + "doh_qname" ], "filters": [ "session_id", "start_timestamp_ms", "end_timestamp_ms", - "duration_ms", - "tcp_handshake_latency_ms", "processing_time", "ingestion_time", "device_id", + "out_link_id", + "in_link_id", "data_center", "sled_ip", "device_group", "address_type", "decoded_as", + "direction", "vsys_id", - "flags", - "flags_identify_info", "proxy_rule_list", "proxy_action", "client_ip", "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", - "imei", - "imsi", - "apn", - "phone_number", - "in_src_mac", - "out_src_mac", "server_ip", "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", - "in_dest_mac", - "out_dest_mac", - "app_path", - "app_debug_info", - "app", - "app_content", - "dup_traffic_flag", - "tunnel_endpoint_a_desc", - "tunnel_endpoint_b_desc", - "protocol_path", - "fqdn_category_list", - "sent_pkts", - "received_pkts", + "ip_protocol", "sent_bytes", "received_bytes", - "tcp_c2s_ip_fragments", - "tcp_s2c_ip_fragments", - "tcp_c2s_lost_bytes", - "tcp_s2c_lost_bytes", - "tcp_c2s_o3_pkts", - "tcp_s2c_o3_pkts", - "tcp_c2s_rtx_pkts", - "tcp_s2c_rtx_pkts", - "tcp_c2s_rtx_bytes", - "tcp_s2c_rtx_bytes", - "tcp_rtt_ms", "tcp_rtt_ms", + "tunnel_id_list", "http_url", "http_host", "http_request_line", @@ -303,12 +218,9 @@ "http_version", "http_status_code", "http_response_latency_ms", - "http_session_duration_ms", "http_action_file_size", "doh_url", "doh_host", - "doh_url", - "doh_host", "doh_request_line", "doh_response_line", "doh_cookie", @@ -335,13 +247,94 @@ "doh_qclass", "doh_cname", "doh_sub", - "doh_rr" + "doh_rr", + "internal_ip_list", + "external_ip_list", + "internal_port_list", + "external_port_list", + "internal_asn_list", + "external_asn_list", + "proxy_rule_id" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" }, "details": { - "$ref": "public_schema_info.json#/details" + "general": { + "$ref": "public_schema_info.json#/details/general" + }, + "treatment": { + "$ref": "public_schema_info.json#/details/treatment" + }, + "source": { + "$ref": "public_schema_info.json#/details/source" + }, + "destination": { + "$ref": "public_schema_info.json#/details/destination" + }, + "application": { + "$ref": "public_schema_info.json#/details/application" + }, + "protocol": [ + "ip_protocol", + "decoded_path", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "doh_url", + "doh_host", + "doh_request_line", + "doh_response_line", + "doh_cookie", + "doh_referer", + "doh_user_agent", + "doh_content_length", + "doh_content_type", + "doh_set_cookie", + "doh_version", + "doh_message_id", + "doh_qr", + "doh_opcode", + "doh_aa", + "doh_tc", + "doh_rd", + "doh_ra", + "doh_rcode", + "doh_qdcount", + "doh_ancount", + "doh_nscount", + "doh_arcount", + "doh_qname", + "doh_qtype", + "doh_qclass", + "doh_cname", + "doh_sub", + "doh_rr" + ], + "transmission": { + "$ref": "public_schema_info.json#/details/transmission" + }, + "other": { + "$ref": "public_schema_info.json#/details/other" + } } }, "decoded_as": { @@ -353,28 +346,29 @@ }, "DoH": { "$ref": "public_schema_info.json#/decoded_as/DoH" - }, - "default_columns": [ - "recv_time", - "subscriber_id", - "client_ip", - "client_port", - "server_ip", - "server_port", - "decoded_as", - "server_fqdn" - ], - "internal_columns": [ - "recv_time", - "log_id", - "flags_identify_info", - "tunnels", - "app_debug_info", - "app_content", - "packet_capture_file", - "tunnel_endpoint_a_desc", - "tunnel_endpoint_b_desc" - ], + } + }, + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "decoded_as", + "server_fqdn" + ], + "internal_columns": [ + "recv_time", + "log_id", + "flags_identify_info", + "encapsulation", + "app_debug_info", + "app_content", + "packet_capture_file", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc" + ], "action_columns": { "intercept": [ "proxy_pinning_status", @@ -388,50 +382,321 @@ "proxy_intercept_error" ] }, - "tunnel_type": { - "$ref": "public_schema_info.json#/tunnel_type" - }, - "measurements": { - "aggregates": { - "sessions": [ - { - "fn": "count", - "column": "log_id", - "value": "sessions", - "label": "Sessions", - "unit": "sessions" - } - ], - "bytes": [ - { - "fn": "sum", - "column": "sent_bytes + received_bytes", - "value": "bytes", - "label": "Bytes", - "unit": "bytes" - } - ], - "incoming_bytes": [ - { - "fn": "sum", - "column": "if(bitAnd(flags, 8) = 8, received_bytes, sent_bytes)", - "value": "incoming_bytes", - "label": "Incoming Bytes", - "unit": "bytes" - } - ], - "outgoing_bytes": [ - { - "fn": "sum", - "column": "if(bitAnd(flags, 8) = 8, sent_bytes, received_bytes)", - "value": "outgoing_bytes", - "label": "Outgoing Bytes", - "unit": "bytes" - } - ] + "tunnel_type": { + "$ref": "public_schema_info.json#/tunnel_type" + }, + "measurements": { + "aggregated_metric_unit": { + "SUM": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes" + }, + "RATE": { + "sent_pkts": "pps", + "received_pkts": "pps", + "sent_bytes": "Bps", + "received_bytes": "Bps", + "tcp_c2s_lost_bytes": "Bps", + "tcp_s2c_lost_bytes": "Bps", + "tcp_c2s_o3_pkts": "pps", + "tcp_s2c_o3_pkts": "pps", + "tcp_c2s_rtx_pkts": "pps", + "tcp_s2c_rtx_pkts": "pps", + "tcp_c2s_rtx_bytes": "Bps", + "tcp_s2c_rtx_bytes": "Bps", + "http_request_content_length": "Bps", + "http_response_content_length": "Bps" + }, + "BITRATE": { + "sent_bytes": "bps", + "received_bytes": "bps", + "tcp_c2s_lost_bytes": "bps", + "tcp_s2c_lost_bytes": "bps", + "tcp_c2s_rtx_bytes": "bps", + "tcp_s2c_rtx_bytes": "bps", + "http_request_content_length": "bps", + "http_response_content_length": "bps" + }, + "MAX": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms" + }, + "MIN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms" + }, + "AVG": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms" + }, + "MEDIAN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms" + }, + "P95_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms" + }, + "P99_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms" } + }, + "field_discovery_metric": { + "sessions": [ + { + "fn": "count", + "column": "log_id", + "value": "sessions", + "label": "Sessions", + "unit": "sessions" + } + ], + "bytes": [ + { + "fn": "sum", + "column": "sent_bytes + received_bytes", + "value": "bytes", + "label": "Bytes", + "unit": "bytes" + } + ], + "incoming_bytes": [ + { + "fn": "sum", + "column": "if(bitAnd(flags, 8) = 8, received_bytes, sent_bytes)", + "value": "incoming_bytes", + "label": "Incoming Bytes", + "unit": "bytes" + } + ], + "outgoing_bytes": [ + { + "fn": "sum", + "column": "if(bitAnd(flags, 8) = 8, sent_bytes, received_bytes)", + "value": "outgoing_bytes", + "label": "Outgoing Bytes", + "unit": "bytes" + } + ] } - } + }, + "expression_fields": [ + { + "name": "internal_ip_list", + "label": "Internal IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_ip,''), if(bitAnd(flags, 16)=16, server_ip,''))" + } + }, + { + "name": "external_ip_list", + "label": "External IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_ip,''), if(bitAnd(flags, 16)!=16, server_ip,''))" + } + }, + { + "name": "internal_port_list", + "label": "Internal Port List", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_port, null), if(bitAnd(flags, 16)=16, server_port, null))" + } + }, + { + "name": "external_port_list", + "label": "External Port List", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_port, null), if(bitAnd(flags, 16)!=16, server_port, null))" + } + }, + { + "name": "internal_asn_list", + "label": "Internal ASN List", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_asn, null), if(bitAnd(flags, 16)=16, server_asn, null))" + } + }, + { + "name": "external_asn_list", + "label": "External ASN List", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_asn, null), if(bitAnd(flags, 16)!=16, server_asn, null))" + } + }, + { + "name": "proxy_rule_id", + "label": "Proxy Rule ID", + "type": "long", + "doc": { + "allow_query": "false", + "expression": "arrayJoin(IF(empty(proxy_rule_list), arrayPushFront(proxy_rule_list, null),proxy_rule_list))" + } + } + ] }, "fields": [ { @@ -453,11 +718,9 @@ "name": "log_id", "label": "Log ID", "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], + "constraints": { + "aggregation_functions": "COUNT" + }, "visibility": "enabled", "ttl": null }, @@ -468,7 +731,7 @@ "label": "Decoded AS", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -480,8 +743,8 @@ "value": "HTTP" }, { - "code": "Doh", - "value": "Doh" + "code": "DoH", + "value": "DoH" } ], "visibility": "enabled", @@ -506,11 +769,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -527,11 +785,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -542,24 +795,26 @@ }, { "name": "duration_ms", - "label": "Duration (ms)", + "label": "Duration(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "int" }, { "name": "tcp_handshake_latency_ms", - "label": "TCP Handshake Latency (ms)", + "label": "TCP Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "int" @@ -571,11 +826,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "current_timestamp" - } - ], "visibility": "enabled", "ttl": null }, @@ -591,15 +841,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "ingestion_time" - }, - { - "function": "get_value", - "appendTo": "recv_time" - } - ], "visibility": "enabled", "ttl": null }, @@ -655,18 +896,6 @@ "label": "Device Tag", "doc": { "visibility": "hidden", - "format": [ - { - "function": "flattenSpec", - "appendTo": "data_center", - "param": "$.tags[?(@.tag=='data_center')][0].value" - }, - { - "function": "flattenSpec", - "appendTo": "device_group", - "param": "$.tags[?(@.tag=='device_group')][0].value" - } - ], "ttl": null }, "type": "string" @@ -676,7 +905,7 @@ "label": "Data Center", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -693,7 +922,7 @@ "label": "Device Group", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -710,6 +939,7 @@ "label": "Sled IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, "visibility": "enabled", @@ -724,22 +954,30 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/address_type/data" + }, "visibility": "enabled", "ttl": null }, "type": "int" }, { + "name": "direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/direction/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { "name": "vsys_id", "label": "Vsys ID", "doc": { @@ -770,7 +1008,7 @@ "data": { "$ref": "public_schema_info.json#/fields/flags/data" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": { @@ -782,12 +1020,36 @@ "name": "flags_identify_info", "label": "Flags Identify Info", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" }, { + "name": "c2s_ttl", + "label": "Client-to-Server TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "hidden", + "ttl": null + }, + "type": "int" + }, + { + "name": "s2c_ttl", + "label": "Server-to-Client TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "hidden", + "ttl": null + }, + "type": "int" + }, + { "name": "security_rule_list", "label": "Security Rule List", "doc": { @@ -809,18 +1071,11 @@ "doc": { "visibility": "hidden", "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/security_action/data" }, - "data": [ - { - "code": "Deny", - "value": "Deny" - }, - { - "code": "Allow", - "value": "Allow" - } - ], "ttl": null }, "type": "string" @@ -858,8 +1113,24 @@ } }, { + "name": "statistics_rule_list", + "label": "Statistics Rule List", + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "hidden", + "ttl": null + }, + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + } + }, + { "name": "sc_rsp_raw", - "label": "Service Chaining Rendered Service Path (Raw)", + "label": "Service Chaining Rendered Service Path(Raw)", "doc": { "constraints": { "operator_functions": "has,notEmpty,empty" @@ -875,7 +1146,7 @@ }, { "name": "sc_rsp_decrypted", - "label": "Service Chaining Rendered Service Path (Decrypted)", + "label": "Service Chaining Rendered Service Path(Decrypted)", "doc": { "constraints": { "operator_functions": "has,notEmpty,empty" @@ -927,7 +1198,7 @@ "doc": { "visibility": "enabled", "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -975,20 +1246,12 @@ "name": "proxy_pinning_status", "label": "Proxy Pinning Status", "doc": { - "data": [ - { - "code": "0", - "value": "not pinning" - }, - { - "code": "1", - "value": "pinning" - }, - { - "code": "2", - "value": "maybe pinning" - } - ], + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/proxy_pinning_status/data" + }, "visibility": "hidden", "ttl": null }, @@ -998,20 +1261,12 @@ "name": "proxy_intercept_status", "label": "Proxy Intercept Status", "doc": { - "data": [ - { - "code": "0", - "value": "passthrough" - }, - { - "code": "1", - "value": "intercept" - }, - { - "code": "2", - "value": "shutdown" - } - ], + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/proxy_intercept_status/data" + }, "visibility": "hidden", "ttl": null }, @@ -1028,7 +1283,7 @@ }, { "name": "proxy_server_side_latency_ms", - "label": "Proxy Server Side Latency", + "label": "Proxy Server-Side Latency(ms)", "doc": { "visibility": "hidden", "ttl": null @@ -1037,7 +1292,7 @@ }, { "name": "proxy_client_side_latency_ms", - "label": "Proxy Client Side Latency", + "label": "Proxy Client-Side Latency(ms)", "doc": { "visibility": "hidden", "ttl": null @@ -1046,7 +1301,7 @@ }, { "name": "proxy_client_side_version", - "label": "Proxy Client Side Version", + "label": "Proxy Client-Side Version", "doc": { "visibility": "hidden", "ttl": null @@ -1055,7 +1310,7 @@ }, { "name": "proxy_server_side_version", - "label": "Proxy Server Side Version", + "label": "Proxy Server-Side Version", "doc": { "visibility": "hidden", "ttl": null @@ -1064,7 +1319,7 @@ }, { "name": "proxy_cert_verify", - "label": "Proxy Cert Verify", + "label": "Proxy Certificate Verify", "doc": { "visibility": "hidden", "ttl": null @@ -1081,8 +1336,8 @@ "type": "string" }, { - "name": "security_mirrored_pkts", - "label": "Security Mirrored Packets", + "name": "monitor_mirrored_pkts", + "label": "Monitor Mirrored Packets", "type": "int", "doc": { "constraints": { @@ -1093,8 +1348,8 @@ } }, { - "name": "security_mirrored_bytes", - "label": "Security Mirrored Bytes", + "name": "monitor_mirrored_bytes", + "label": "Monitor Mirrored Bytes", "type": "int", "doc": { "constraints": { @@ -1109,18 +1364,9 @@ "label": "Client IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "client_asn" - }, - { - "function": "radius_match", - "appendTo": "subscriber_id" - } - ], "visibility": "enabled", "ttl": null }, @@ -1131,6 +1377,10 @@ "label": "Client Port", "doc": { "visibility": "enabled", + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -1139,14 +1389,59 @@ "name": "client_os_desc", "type": "string", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "label": "Client OS Description" }, { "name": "client_geolocation", - "label": "Client Geolocation", + "label": "Client Geolocation(Deprecated)", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_country", + "label": "Client Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_super_administrative_area", + "label": "Client Super Administrative Area", + "doc": { + "constraints": { + "type": "super_administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_administrative_area", + "label": "Client Administrative Area", + "doc": { + "constraints": { + "type": "administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_sub_administrative_area", + "label": "Client SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1158,6 +1453,9 @@ "label": "Client ASN", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "long" @@ -1175,7 +1473,7 @@ "name": "imei", "label": "IMEI", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1184,7 +1482,7 @@ "name": "imsi", "label": "IMSI", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1193,7 +1491,7 @@ "name": "apn", "label": "APN", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1202,7 +1500,7 @@ "name": "phone_number", "label": "Phone Number", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1212,14 +1510,9 @@ "label": "Server IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "server_asn" - } - ], "visibility": "enabled", "ttl": null }, @@ -1230,6 +1523,10 @@ "label": "Server Port", "doc": { "visibility": "enabled", + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -1238,14 +1535,59 @@ "name": "server_os_desc", "type": "string", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "label": "Server OS Description" }, { "name": "server_geolocation", - "label": "Server Geolocation", + "label": "Server Geolocation(Deprecated)", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_country", + "label": "Server Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_super_administrative_area", + "label": "Server Super Administrative Area", + "doc": { + "constraints": { + "type": "super_administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_administrative_area", + "label": "Server Administrative Area", + "doc": { + "constraints": { + "type": "administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_sub_administrative_area", + "label": "Server SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1257,6 +1599,9 @@ "label": "Server ASN", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "long" @@ -1265,6 +1610,9 @@ "name": "server_fqdn", "label": "Server FQDN", "doc": { + "constraints": { + "type": "fqdn" + }, "visibility": "enabled", "ttl": null }, @@ -1274,17 +1622,20 @@ "name": "server_domain", "label": "Server Domain", "doc": { + "constraints": { + "type": "domain" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "app_path", - "label": "Application Path", + "name": "app_transition", + "label": "Application Transition", "type": "string", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null } }, @@ -1292,7 +1643,19 @@ "name": "app", "label": "Application", "doc": { - "visibility": "enabled", + "constraints": { + "type": "app" + }, + "visibility": "hidden", + "ttl": null + }, + "type": "string" + }, + { + "name": "app_category", + "label": "Application Category", + "doc": { + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1301,7 +1664,7 @@ "name": "app_debug_info", "label": "Application Debug Info", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1310,21 +1673,45 @@ "name": "app_content", "label": "Application Content", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" }, { - "name": "protocol_path", - "label": "Protocol Path", + "name": "app_extra_info", + "label": "Application Extra Info", "doc": { + "visibility": "hidden", + "ttl": null + }, + "type": "string" + }, + { + "name": "ip_protocol", + "label": "IP Protocol", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/ip_protocol/data" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { + "name": "decoded_path", + "label": "Decoded Path", + "doc": { + "visibility": "hidden", + "ttl": null + }, + "type": "string" + }, + { "name": "fqdn_category_list", "label": "FQDN Category List", "doc": { @@ -1332,16 +1719,16 @@ "operator_functions": "has,notEmpty,empty" }, "dict_location": { - "path": "/v1/category/dict", - "key": "categoryId", - "value": "categoryName" + "path": "/v1/policy/object?type=fqdn_category", + "key": "category_id", + "value": "category_name" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": { "type": "array", - "items": "int", + "items": "long", "logicalType": "array" } }, @@ -1352,7 +1739,7 @@ "constraints": { "type": "decimal" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" @@ -1364,7 +1751,7 @@ "constraints": { "type": "decimal" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" @@ -1395,139 +1782,140 @@ }, { "name": "tcp_c2s_ip_fragments", - "label": "TCP C2S IP Fragments ", + "label": "Client-to-Server IP Fragments", "doc": { "constraints": { "type": "decimal" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_s2c_ip_fragments", - "label": "TCP S2C IP Fragments", + "label": "Server-to-Client IP Fragments", "doc": { "constraints": { "type": "decimal" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_c2s_lost_bytes", - "label": "TCP C2S Lost Bytes", + "label": "Client-to-Server Lost Bytes", "doc": { "constraints": { "type": "bytes" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_s2c_lost_bytes", - "label": "TCP S2C Lost Bytes", + "label": "Server-to-Client Lost Bytes", "doc": { "constraints": { "type": "bytes" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_c2s_o3_pkts", - "label": "TCP C2S out-of-order packets", + "label": "Client-to-Server Out-of-Order Packets", "doc": { "constraints": { "type": "decimal" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_s2c_o3_pkts", - "label": "TCP S2C out-of-order packets", + "label": "Server-to-Client Out-of-Order Packets", "doc": { "constraints": { "type": "decimal" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_c2s_rtx_pkts", - "label": "TCP C2S Retransmission Packets", + "label": "Client-to-Server Retransmission Packets", "doc": { "constraints": { "type": "decimal" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_s2c_rtx_pkts", - "label": "TCP S2C Retransmission Packets", + "label": "Server-to-Client Retransmission Packets", "doc": { "constraints": { "type": "decimal" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_c2s_rtx_bytes", - "label": "TCP C2S Retransmission Bytes ", + "label": "Client-to-Server Retransmission Bytes", "doc": { "constraints": { "type": "bytes" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_s2c_rtx_bytes", - "label": "TCP S2C Retransmission Bytes", + "label": "Server-to-Client Retransmission Bytes", "doc": { "constraints": { "type": "bytes" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "long" }, { "name": "tcp_rtt_ms", - "label": "Round-trip Time", + "label": "Round-trip Time(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "int" }, { "name": "tcp_client_isn", - "label": "TCP Client ISN", + "label": "Client ISN", "doc": { "visibility": "hidden", "ttl": null @@ -1536,7 +1924,7 @@ }, { "name": "tcp_server_isn", - "label": "TCP Server ISN", + "label": "Server ISN", "doc": { "visibility": "hidden", "ttl": null @@ -1548,7 +1936,7 @@ "label": "Packet Capture File", "doc": { "allow_query": "false", - "visibility": "enabled", + "visibility": "hidden", "constraints": { "type": "file" }, @@ -1560,7 +1948,7 @@ "name": "in_src_mac", "label": "Incoming Source MAC", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1569,7 +1957,7 @@ "name": "out_src_mac", "label": "Outgoing Source MAC", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1578,7 +1966,7 @@ "name": "in_dest_mac", "label": "Incoming Destination MAC", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1587,24 +1975,17 @@ "name": "out_dest_mac", "label": "Outgoing Destination MAC", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" }, { - "name": "tunnels", - "label": "Tunnels", + "name": "encapsulation", + "label": "Encapsulation", "doc": { - "visibility": "enabled", - "ttl": null, - "format": [ - { - "function": "gtpc_match", - "appendTo": "imsi,imei,phone_number", - "param": "$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_a2b_teid,$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_b2a_teid" - } - ] + "visibility": "hidden", + "ttl": null }, "type": "string" }, @@ -1618,25 +1999,38 @@ "data": { "$ref": "public_schema_info.json#/fields/dup_traffic_flag/data" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "int" }, { + "name": "tunnel_id_list", + "label": "Tunnel ID List", + "doc": { + "visibility": "hidden", + "ttl": null + }, + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + } + }, + { "name": "tunnel_endpoint_a_desc", - "label": "Tunnel Endpoint A Description", + "label": "Tunnel Endpoint A Description(Deprecated)", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" }, { "name": "tunnel_endpoint_b_desc", - "label": "Tunnel Endpoint B Description", + "label": "Tunnel Endpoint B Description(Deprecated)", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1645,6 +2039,9 @@ "name": "http_url", "label": "HTTP.URL", "doc": { + "constraints": { + "type": "url" + }, "visibility": "enabled", "ttl": null }, @@ -1679,16 +2076,16 @@ }, { "name": "http_request_content_length", - "label": "HTTP.Request Content Length", + "label": "HTTP.Request Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_request_content_type", - "label": "HTTP.Request Content Type", + "label": "HTTP.Request Content-Type", "doc": { "visibility": "enabled", "ttl": null @@ -1697,12 +2094,12 @@ }, { "name": "http_response_content_length", - "label": "HTTP.Response Content Length", + "label": "HTTP.Response Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_response_content_type", @@ -1717,11 +2114,6 @@ "name": "http_request_body", "label": "HTTP.Request Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -1735,11 +2127,6 @@ "name": "http_response_body", "label": "HTTP.Response Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -1753,7 +2140,7 @@ "name": "http_proxy_flag", "label": "HTTP.Proxy Flag", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "int" @@ -1787,7 +2174,7 @@ }, { "name": "http_user_agent", - "label": "HTTP.User Agent", + "label": "HTTP.User-Agent", "doc": { "visibility": "enabled", "ttl": null @@ -1796,7 +2183,7 @@ }, { "name": "http_set_cookie", - "label": "HTTP.Set Cookie", + "label": "HTTP.Set-Cookie", "doc": { "visibility": "enabled", "ttl": null @@ -1817,21 +2204,26 @@ "label": "HTTP.Status Code", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT", + "type": "http_code" + }, "ttl": null }, "type": "int" }, { "name": "http_response_latency_ms", - "label": "HTTP.Response Latency (ms)", + "label": "HTTP.Response Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "http_action_file_size", @@ -1843,19 +2235,20 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "http_session_duration_ms", - "label": "HTTP.Session Duration (ms)", + "label": "HTTP.Session Duration(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "doh_url", @@ -1913,7 +2306,7 @@ }, { "name": "doh_user_agent", - "label": "DoH.User Agent", + "label": "DoH.User-Agent", "doc": { "visibility": "enabled", "ttl": null @@ -1963,7 +2356,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_qr", @@ -1974,18 +2367,18 @@ }, "data": [ { - "code": "0", + "code": 0, "value": "QUERY" }, { - "code": "1", + "code": 1, "value": "REESPONSE" } ], "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_opcode", @@ -1996,26 +2389,26 @@ }, "data": [ { - "code": "0", + "code": 0, "value": "QUERY" }, { - "code": "1", + "code": 1, "value": "IQUERY" }, { - "code": "2", + "code": 2, "value": "STATUS" }, { - "code": "5", + "code": 5, "value": "UPDATE" } ], "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_aa", @@ -2024,7 +2417,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_tc", @@ -2033,7 +2426,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_rd", @@ -2042,7 +2435,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_ra", @@ -2051,7 +2444,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_rcode", @@ -2060,7 +2453,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_qdcount", @@ -2072,7 +2465,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_ancount", @@ -2084,7 +2477,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_nscount", @@ -2096,7 +2489,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_arcount", @@ -2108,7 +2501,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_qname", @@ -2128,50 +2521,50 @@ }, "data": [ { - "code": "1", + "code": 1, "value": "A" }, { - "code": "2", + "code": 2, "value": "NS" }, { - "code": "5", + "code": 5, "value": "CNAME" }, { - "code": "6", + "code": 6, "value": "SOA" }, { - "code": "11", + "code": 11, "value": "WKS" }, { - "code": "12", + "code": 12, "value": "PTR" }, { - "code": "13", + "code": 13, "value": "HINFO" }, { - "code": "11", + "code": 11, "value": "WKS" }, { - "code": "15", + "code": 15, "value": "MX" }, { - "code": "28", + "code": 28, "value": "AAAA" } ], "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_qclass", @@ -2180,7 +2573,7 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_cname", @@ -2200,18 +2593,18 @@ }, "data": [ { - "code": "1", + "code": 1, "value": "DNS" }, { - "code": "2", + "code": 2, "value": "DNSSEC" } ], "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "doh_rr", diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/proxy_rule_hits.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/proxy_rule_hits.json index c62f9c37..a31ccdac 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/proxy_rule_hits.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/proxy_rule_hits.json @@ -8,25 +8,17 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "rule_id", - "pinning_status", "action", - "sub_action" + "sub_action", + "device_group", + "data_center" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "rule_id", - "pinning_status", - "action", - "sub_action", "hit_count", "in_bytes", "out_bytes", @@ -34,27 +26,40 @@ "out_pkts" ], "filters": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "rule_id", - "pinning_status", "action", "sub_action", - "hit_count", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" + "pinning_status", + "device_group", + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 1, - "ingestion_delay": 15 + "ingestion_period": "PT1S", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "hit_count": "sessions", + "in_bytes": "bytes", + "out_bytes": "bytes", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "hit_count": "sps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -78,7 +83,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -135,22 +140,22 @@ { "name": "action", "label": "Action", - "type": "int", + "type": "long", "doc": { "constraints": { "operator_functions": "=,in" }, "data": [ { - "code": "2", + "code": 2, "value": "Intercept" }, { - "code": "3", + "code": 3, "value": "No Intercept" }, { - "code": "48", + "code": 48, "value": "Manipulate" } ], @@ -235,6 +240,10 @@ "name": "hit_count", "label": "Hit Count", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -243,6 +252,10 @@ "name": "in_bytes", "label": "Incoming Bytes", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -251,6 +264,10 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -259,6 +276,10 @@ "name": "in_pkts", "label": "Incoming Packets", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -267,6 +288,10 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/public_schema_info.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/public_schema_info.json index 5ebafa86..7920ca94 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/public_schema_info.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/public_schema_info.json @@ -4,47 +4,135 @@ {
"name": "COUNT",
"label": "COUNT",
- "function": "count(expr)"
+ "function": "count(expr)",
+ "metric_type": "counter",
+ "unit": "short"
},
{
"name": "COUNT_DISTINCT",
"label": "COUNT_DISTINCT",
- "function": "count(distinct expr)"
+ "function": "COUNT_DISTINCT(expr)",
+ "metric_type": "gauge",
+ "unit": "short"
},
{
"name": "AVG",
"label": "AVG",
- "function": "avg(expr)"
+ "function": "avg(expr)",
+ "metric_type": "gauge",
+ "unit": "short"
},
{
"name": "SUM",
"label": "SUM",
- "function": "sum(expr)"
+ "function": "sum(expr)",
+ "metric_type": "counter",
+ "unit": "short"
},
{
"name": "MAX",
"label": "MAX",
- "function": "max(expr)"
+ "function": "max(expr)",
+ "metric_type": "gauge",
+ "unit": "short"
},
{
"name": "MIN",
"label": "MIN",
- "function": "min(expr)"
+ "function": "min(expr)",
+ "metric_type": "gauge",
+ "unit": "short"
},
{
"name": "MEDIAN",
"label": "MEDIAN",
- "function": "MEDIAN(expr)"
+ "function": "MEDIAN(expr)",
+ "metric_type": "gauge",
+ "unit": "short"
+ },
+ {
+ "name": "MEDIAN_HDR",
+ "label": "MEDIAN_HDR",
+ "function": "MEDIAN_HDR(expr)",
+ "metric_type": "gauge",
+ "unit": "short"
},
{
"name": "QUANTILE",
"label": "QUANTILE",
- "function": "QUANTILE(expr,level)"
+ "function": "QUANTILE(expr, level)",
+ "metric_type": "gauge",
+ "unit": "short"
},
{
- "name": "TIME_FLOOR_WITH_FILL",
- "label": "TIME_FLOOR_WITH_FILL",
- "function": "TIME_FLOOR_WITH_FILL(expr,period,fill)"
+ "name": "P95_PERCENTILE",
+ "label": "P95_PERCENTILE",
+ "function": "QUANTILE(expr, 0.95)",
+ "metric_type": "gauge",
+ "unit": "short"
+ },
+ {
+ "name": "P99_PERCENTILE",
+ "label": "P99_PERCENTILE",
+ "function": "QUANTILE(expr, 0.99)",
+ "metric_type": "gauge",
+ "unit": "short"
+ },
+ {
+ "name": "QUANTILE_HDR",
+ "label": "QUANTILE_HDR",
+ "function": "QUANTILE_HDR(expr, level)",
+ "metric_type": "gauge",
+ "unit": "short"
+ },
+ {
+ "name": "P95_PERCENTILE_HDR",
+ "label": "P95_PERCENTILE_HDR",
+ "function": "QUANTILE_HDR(expr, 0.95)",
+ "metric_type": "gauge",
+ "unit": "short"
+ },
+ {
+ "name": "P99_PERCENTILE_HDR",
+ "label": "P99_PERCENTILE_HDR",
+ "function": "QUANTILE_HDR(expr, 0.99)",
+ "metric_type": "gauge",
+ "unit": "short"
+ },
+ {
+ "name": "PERCENTILES_HDR",
+ "label": "PERCENTILES_HDR",
+ "function": "PERCENTILES_HDR(expr)",
+ "metric_type": "histogram",
+ "unit": "short"
+ },
+ {
+ "name": "APPROX_COUNT_DISTINCT_HLLD",
+ "label": "COUNT_DISTINCT_HLLD",
+ "function": "APPROX_COUNT_DISTINCT_HLLD(expr)",
+ "metric_type": "gauge",
+ "unit": "short"
+ },
+ {
+ "name": "APPROX_COUNT_DISTINCT_DS_HLL",
+ "label": "COUNT_DISTINCT_DS_HLL",
+ "function": "APPROX_COUNT_DISTINCT_DS_HLL(expr)",
+ "metric_type": "gauge",
+ "unit": "short"
+ },
+ {
+ "name": "RATE",
+ "label": "RATE",
+ "function": "RATE(<expr>, <duration>, [interval])",
+ "metric_type": "gauge",
+ "unit": "sps"
+ },
+ {
+ "name": "BITRATE",
+ "label": "BITRATE",
+ "function": "RATE(<expr>, <duration>, [interval]) * 8",
+ "metric_type": "gauge",
+ "unit": "bps"
}
],
"date": [
@@ -54,19 +142,34 @@ "function": "UNIX_TIMESTAMP(expr)"
},
{
+ "name": "UNIX_TIMESTAMP_MILLIS",
+ "label": "UNIX_TIMESTAMP_MILLIS",
+ "function": "UNIX_TIMESTAMP_MILLIS(expr)"
+ },
+ {
"name": "FROM_UNIXTIME",
"label": "FROM_UNIXTIME",
"function": "FROM_UNIXTIME(expr)"
},
{
+ "name": "FROM_UNIXTIME_MILLIS",
+ "label": "FROM_UNIXTIME_MILLIS",
+ "function": "FROM_UNIXTIME_MILLIS(expr)"
+ },
+ {
"name": "DATE_FORMAT",
"label": "DATE_FORMAT",
- "function": "DATE_FORMAT(expr,format)"
+ "function": "DATE_FORMAT(expr, format)"
},
{
"name": "CONVERT_TZ",
"label": "CONVERT_TZ",
"function": "CONVERT_TZ(expr, from_tz, to_tz)"
+ },
+ {
+ "name": "TIME_FLOOR_WITH_FILL",
+ "label": "TIME_FLOOR_WITH_FILL",
+ "function": "TIME_FLOOR_WITH_FILL(expr, period, fill)"
}
],
"operator": [
@@ -136,136 +239,45 @@ "function": "empty(expr)"
},
{
- "name": "bitAnd",
- "label": "Bitwise AND",
- "function": "bitAnd(expr, value)=value"
- }
- ],
- "sketch": [
- {
- "name": "APPROX_COUNT_DISTINCT_HLLD",
- "label": "APPROX_COUNT_DISTINCT_HLLD",
- "function": "APPROX_COUNT_DISTINCT_HLLD(expr, precision,round)"
- },
- {
- "name": "HLLD",
- "label": "HLLD",
- "function": "HLLD(expr, precision)"
- },
- {
- "name": "APPROX_QUANTILE_HDR",
- "label": "APPROX_QUANTILE_HDR",
- "function": "APPROX_QUANTILE_HDR(expr,probability,numberOfSignificantValueDigits)"
+ "name": "is null",
+ "label": "IS NULL",
+ "function": "expr is null"
},
{
- "name": "HDR_HISTOGRAM",
- "label": "HDR_HISTOGRAM",
- "function": "HDR_HISTOGRAM(expr, numberOfSignificantValueDigits)"
+ "name": "is not null",
+ "label": "IS NOT NULL",
+ "function": "expr is not null"
},
{
- "name": "HDR_GET_QUANTILES",
- "label": "HDR_GET_QUANTILES",
- "function": "HDR_GET_QUANTILES(expr,probability0, probability1, ...)"
+ "name": "bitAnd",
+ "label": "Bitwise AND",
+ "function": "bitAnd(expr, value) = value"
},
{
- "name": "HDR_GET_PERCENTILES",
- "label": "HDR_GET_PERCENTILES",
- "function": "HDR_GET_QUANTILES(expr,percentileTicksPerHalfDistance)"
+ "name": "isIPAddressInRange",
+ "label": "INCIDR",
+ "function": "isIPAddressInRange(expr, prefix)"
}
]
},
- "session_record": {
- "index_key": [
- "vsys_id",
- "security_action",
- "proxy_action",
- "decoded_as",
- "data_center",
- "device_group",
- "recv_time"
- ]
- },
- "security_event": {
- "index_key": [
- "vsys_id",
- "security_action",
- "proxy_action",
- "decoded_as",
- "data_center",
- "device_group",
- "recv_time"
- ]
- },
- "proxy_event": {
- "index_key": [
- "vsys_id",
- "proxy_action",
- "decoded_as",
- "data_center",
- "device_group",
- "recv_time"
- ]
- },
- "monitor_event": {
- "index_key": [
- "vsys_id",
- "security_action",
- "proxy_action",
- "decoded_as",
- "data_center",
- "device_group",
- "recv_time"
- ]
- },
- "transaction_record": {
- "index_key": [
- "vsys_id",
- "session_id",
- "recv_time"
- ]
- },
- "voip_record": {
- "index_key": [
- "vsys_id",
- "decoded_as",
- "data_center",
- "device_group",
- "recv_time"
- ]
- },
- "dos_event": {
- "index_key": [
- "vsys_id",
- "destination_ip",
- "recv_time",
- "log_id"
- ]
- },
- "assessment_event": {
- "index_key": [
- "vsys_id",
- "recv_time",
- "log_id"
- ]
- },
"schema_query": {
"references": {
"aggregation": [
{
"type": "int",
- "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,QUANTILE"
+ "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE"
},
{
"type": "long",
- "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,QUANTILE"
+ "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE"
},
{
"type": "float",
- "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,QUANTILE"
+ "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE"
},
{
"type": "double",
- "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,QUANTILE"
+ "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE"
},
{
"type": "string",
@@ -299,11 +311,11 @@ "operator": [
{
"type": "int",
- "functions": "=,!=,>,<,>=,<=,in,not in"
+ "functions": "=,!=,>,<,>=,<=,in,not in,is null,is not null"
},
{
"type": "long",
- "functions": "=,!=,>,<,>=,<=,in,not in"
+ "functions": "=,!=,>,<,>=,<=,in,not in,is null,is not null"
},
{
"type": "float",
@@ -366,15 +378,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -388,12 +404,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -404,14 +424,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -435,8 +462,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc"
],
@@ -470,15 +498,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -492,12 +524,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -508,14 +544,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -539,8 +582,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"http_url",
@@ -596,15 +640,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -618,12 +666,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -634,14 +686,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -665,8 +724,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"mail_protocol_type",
@@ -682,7 +742,8 @@ "mail_subject_charset",
"mail_attachment_name",
"mail_attachment_name_charset",
- "mail_eml_file"
+ "mail_eml_file",
+ "mail_starttls_flag"
],
"default_columns": [
"recv_time",
@@ -717,15 +778,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -739,12 +804,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -755,14 +824,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -786,8 +862,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"dns_message_id",
@@ -842,15 +919,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -864,12 +945,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -880,14 +965,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -911,8 +1003,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"ssl_version",
@@ -958,15 +1051,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -980,12 +1077,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -996,14 +1097,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -1027,8 +1135,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"dtls_cookie",
@@ -1073,15 +1182,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -1095,12 +1208,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -1111,14 +1228,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -1142,8 +1266,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"quic_version",
@@ -1181,15 +1306,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -1203,12 +1332,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -1219,14 +1352,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -1250,8 +1390,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"ftp_account",
@@ -1289,15 +1430,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -1311,12 +1456,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -1327,14 +1476,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -1358,8 +1514,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"sip_call_id",
@@ -1410,15 +1567,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -1432,12 +1593,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -1448,14 +1613,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -1479,8 +1651,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"rtp_payload_type_c2s",
@@ -1520,15 +1693,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -1542,12 +1719,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -1557,14 +1738,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -1588,8 +1776,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"ssh_version",
@@ -1635,15 +1824,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -1657,12 +1850,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -1673,14 +1870,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -1704,8 +1908,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"stratum_cryptocurrency",
@@ -1746,15 +1951,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -1768,12 +1977,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -1784,14 +1997,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -1815,8 +2035,9 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"rdp_cookie",
@@ -1863,15 +2084,19 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"security_rule_list",
"security_action",
"monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -1885,12 +2110,16 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -1901,14 +2130,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -1932,14 +2168,13 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc",
"doh_url",
"doh_host",
- "doh_url",
- "doh_host",
"doh_request_line",
"doh_response_line",
"doh_cookie",
@@ -1978,6 +2213,149 @@ "doh_qname",
"server_port"
]
+ },
+ "VoIP": {
+ "columns": [
+ "recv_time",
+ "log_id",
+ "decoded_as",
+ "session_id",
+ "start_timestamp_ms",
+ "end_timestamp_ms",
+ "duration_ms",
+ "tcp_handshake_latency_ms",
+ "ingestion_time",
+ "processing_time",
+ "insert_time",
+ "device_id",
+ "out_link_id",
+ "in_link_id",
+ "device_tag",
+ "data_center",
+ "device_group",
+ "sled_ip",
+ "address_type",
+ "direction",
+ "vsys_id",
+ "t_vsys_id",
+ "flags",
+ "flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
+ "security_rule_list",
+ "security_action",
+ "monitor_rule_list",
+ "shaping_rule_list",
+ "sc_rule_list",
+ "statistics_rule_list",
+ "sc_rsp_raw",
+ "sc_rsp_decrypted",
+ "proxy_rule_list",
+ "proxy_action",
+ "proxy_pinning_status",
+ "proxy_intercept_status",
+ "proxy_passthrough_reason",
+ "proxy_client_side_latency_ms",
+ "proxy_server_side_latency_ms",
+ "proxy_client_side_version",
+ "proxy_server_side_version",
+ "proxy_cert_verify",
+ "proxy_intercept_error",
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes",
+ "client_ip",
+ "client_port",
+ "client_os_desc",
+ "client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
+ "client_asn",
+ "subscriber_id",
+ "imei",
+ "imsi",
+ "apn",
+ "phone_number",
+ "server_ip",
+ "server_port",
+ "server_os_desc",
+ "server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
+ "server_asn",
+ "server_fqdn",
+ "server_domain",
+ "app_transition",
+ "app",
+ "app_category",
+ "app_debug_info",
+ "app_content",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
+ "fqdn_category_list",
+ "sent_pkts",
+ "received_pkts",
+ "sent_bytes",
+ "received_bytes",
+ "tcp_c2s_ip_fragments",
+ "tcp_s2c_ip_fragments",
+ "tcp_c2s_lost_bytes",
+ "tcp_s2c_lost_bytes",
+ "tcp_c2s_o3_pkts",
+ "tcp_s2c_o3_pkts",
+ "tcp_c2s_rtx_pkts",
+ "tcp_s2c_rtx_pkts",
+ "tcp_c2s_rtx_bytes",
+ "tcp_s2c_rtx_bytes",
+ "tcp_rtt_ms",
+ "tcp_client_isn",
+ "tcp_server_isn",
+ "packet_capture_file",
+ "in_src_mac",
+ "out_src_mac",
+ "in_dest_mac",
+ "out_dest_mac",
+ "encapsulation",
+ "dup_traffic_flag",
+ "tunnel_id_list",
+ "tunnel_endpoint_a_desc",
+ "tunnel_endpoint_b_desc",
+ "sip_call_id",
+ "sip_originator_description",
+ "sip_responder_description",
+ "sip_user_agent",
+ "sip_server",
+ "sip_originator_sdp_connect_ip",
+ "sip_originator_sdp_media_port",
+ "sip_originator_sdp_media_type",
+ "sip_originator_sdp_content",
+ "sip_responder_sdp_connect_ip",
+ "sip_responder_sdp_media_port",
+ "sip_responder_sdp_media_type",
+ "sip_responder_sdp_content",
+ "sip_duration_s",
+ "sip_bye",
+ "rtp_payload_type_c2s",
+ "rtp_payload_type_s2c",
+ "rtp_pcap_path",
+ "rtp_originator_dir"
+ ],
+ "default_columns": [
+ "recv_time",
+ "client_ip",
+ "client_port",
+ "server_ip",
+ "server_port",
+ "sip_originator_description",
+ "sip_responder_description",
+ "sip_call_id",
+ "rtp_pcap_path",
+ "rtp_originator_dir"
+ ]
}
},
"data_view": {
@@ -2002,10 +2380,13 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
"flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl",
"proxy_rule_list",
"proxy_action",
"proxy_pinning_status",
@@ -2021,6 +2402,10 @@ "client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -2031,14 +2416,21 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
"server_domain",
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "protocol_path",
+ "app_extra_info",
+ "ip_protocol",
+ "decoded_path",
"fqdn_category_list",
"sent_pkts",
"received_pkts",
@@ -2062,10 +2454,8 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
+ "encapsulation",
"dup_traffic_flag",
- "tunnel_endpoint_a_desc",
- "tunnel_endpoint_b_desc",
"http_url",
"http_host",
"http_request_line",
@@ -2149,7 +2539,7 @@ "MPLS": [
{
"name": "mpls_c2s_direction_label",
- "label": "Multiprotocol Label (c2s)",
+ "label": "Multiprotocol Label(c2s)",
"type": {
"type": "array",
"items": "int",
@@ -2158,7 +2548,7 @@ },
{
"name": "mpls_s2c_direction_label",
- "label": "Multiprotocol Label (s2c)",
+ "label": "Multiprotocol Label(s2c)",
"type": {
"type": "array",
"items": "int",
@@ -2169,7 +2559,7 @@ "VLAN": [
{
"name": "vlan_c2s_direction_id",
- "label": "VLAN Direction (c2s)",
+ "label": "VLAN Direction(c2s)",
"type": {
"type": "array",
"items": "int",
@@ -2178,7 +2568,7 @@ },
{
"name": "vlan_s2c_direction_id",
- "label": "VLAN Direction (s2c)",
+ "label": "VLAN Direction(s2c)",
"type": {
"type": "array",
"items": "int",
@@ -2201,22 +2591,22 @@ "MULTIPATH_ETHERNET": [
{
"name": "c2s_source_mac",
- "label": "Source MAC (c2s)",
+ "label": "Source MAC(c2s)",
"type": "string"
},
{
"name": "c2s_destination_mac",
- "label": "Destination MAC (c2s)",
+ "label": "Destination MAC(c2s)",
"type": "string"
},
{
"name": "s2c_source_mac",
- "label": "Source MAC (s2c)",
+ "label": "Source MAC(s2c)",
"type": "string"
},
{
"name": "s2c_destination_mac",
- "label": "Destination MAC (s2c)",
+ "label": "Destination MAC(s2c)",
"type": "string"
}
],
@@ -2278,7 +2668,32 @@ "label": "Down Tunnel ID",
"type": "int"
}
- ]
+ ],
+ "IPv4": [
+ {
+ "name": "client_ip",
+ "label": "Client IP",
+ "type": "string"
+ },
+ {
+ "name": "server_ip",
+ "label": "Server IP",
+ "type": "string"
+ }
+ ],
+ "IPv6": [
+ {
+ "name": "client_ip",
+ "label": "Client IP",
+ "type": "string"
+ },
+ {
+ "name": "server_ip",
+ "label": "Server IP",
+ "type": "string"
+ }
+ ],
+ "GRE": []
},
"details": {
"general": [
@@ -2301,10 +2716,13 @@ "device_group",
"sled_ip",
"address_type",
+ "direction",
"vsys_id",
"t_vsys_id",
"flags",
- "flags_identify_info"
+ "flags_identify_info",
+ "c2s_ttl",
+ "s2c_ttl"
],
"treatment": [
"security_rule_list",
@@ -2312,6 +2730,7 @@ "monitor_rule_list",
"shaping_rule_list",
"sc_rule_list",
+ "statistics_rule_list",
"sc_rsp_raw",
"sc_rsp_decrypted",
"proxy_rule_list",
@@ -2325,14 +2744,18 @@ "proxy_server_side_version",
"proxy_cert_verify",
"proxy_intercept_error",
- "security_mirrored_pkts",
- "security_mirrored_bytes"
+ "monitor_mirrored_pkts",
+ "monitor_mirrored_bytes"
],
"source": [
"client_ip",
"client_port",
"client_os_desc",
"client_geolocation",
+ "client_country",
+ "client_super_administrative_area",
+ "client_administrative_area",
+ "client_sub_administrative_area",
"client_asn",
"subscriber_id",
"imei",
@@ -2345,149 +2768,22 @@ "server_port",
"server_os_desc",
"server_geolocation",
+ "server_country",
+ "server_super_administrative_area",
+ "server_administrative_area",
+ "server_sub_administrative_area",
"server_asn",
"server_fqdn",
- "server_domain"
+ "server_domain",
+ "fqdn_category_list"
],
"application": [
- "app_path",
+ "app_transition",
"app",
+ "app_category",
"app_debug_info",
"app_content",
- "fqdn_category_list"
- ],
- "protocol": [
- "protocol_path",
- "dns_message_id",
- "dns_qr",
- "dns_opcode",
- "dns_aa",
- "dns_tc",
- "dns_rd",
- "dns_ra",
- "dns_rcode",
- "dns_qdcount",
- "dns_ancount",
- "dns_nscount",
- "dns_arcount",
- "dns_qname",
- "dns_qtype",
- "dns_qclass",
- "dns_cname",
- "dns_sub",
- "dns_rr",
- "dns_response_latency_ms",
- "dtls_cookie",
- "dtls_version",
- "dtls_sni",
- "dtls_san",
- "dtls_cn",
- "dtls_handshake_latency_ms",
- "dtls_ja3_fingerprint",
- "dtls_ja3_hash",
- "dtls_cert_issuer",
- "dtls_cert_subject",
- "ftp_account",
- "ftp_url",
- "ftp_link_type",
- "http_url",
- "http_host",
- "http_request_line",
- "http_response_line",
- "http_request_content_length",
- "http_request_content_type",
- "http_response_content_length",
- "http_response_content_type",
- "http_request_body",
- "http_response_body",
- "http_proxy_flag",
- "http_sequence",
- "http_cookie",
- "http_referer",
- "http_user_agent",
- "http_set_cookie",
- "http_version",
- "http_status_code",
- "http_response_latency_ms",
- "http_session_duration_ms",
- "http_action_file_size",
- "mail_protocol_type",
- "mail_account",
- "mail_from_cmd",
- "mail_to_cmd",
- "mail_from",
- "mail_password",
- "mail_to",
- "mail_cc",
- "mail_bcc",
- "mail_subject",
- "mail_subject_charset",
- "mail_attachment_name",
- "mail_attachment_name_charset",
- "mail_eml_file",
- "quic_version",
- "quic_sni",
- "quic_user_agent",
- "rdp_cookie",
- "rdp_security_protocol",
- "rdp_client_channels",
- "rdp_keyboard_layout",
- "rdp_client_version",
- "rdp_client_name",
- "rdp_client_product_id",
- "rdp_desktop_width",
- "rdp_desktop_height",
- "rdp_requested_color_depth",
- "rdp_certificate_type",
- "rdp_certificate_count",
- "rdp_certificate_permanent",
- "rdp_encryption_level",
- "rdp_encryption_method",
- "ssh_version",
- "ssh_auth_success",
- "ssh_client_version",
- "ssh_server_version",
- "ssh_cipher_alg",
- "ssh_mac_alg",
- "ssh_compression_alg",
- "ssh_kex_alg",
- "ssh_host_key_alg",
- "ssh_host_key",
- "ssh_hassh",
- "ssl_version",
- "ssl_sni",
- "ssl_san",
- "ssl_cn",
- "ssl_handshake_latency_ms",
- "ssl_ja3_hash",
- "ssl_ja3s_hash",
- "ssl_cert_issuer",
- "ssl_cert_subject",
- "ssl_esni_flag",
- "ssl_ech_flag",
- "sip_call_id",
- "sip_originator_description",
- "sip_responder_description",
- "sip_user_agent",
- "sip_server",
- "sip_originator_sdp_connect_ip",
- "sip_originator_sdp_media_port",
- "sip_originator_sdp_media_type",
- "sip_originator_sdp_content",
- "sip_responder_sdp_connect_ip",
- "sip_responder_sdp_media_port",
- "sip_responder_sdp_media_type",
- "sip_responder_sdp_content",
- "sip_duration_s",
- "sip_bye",
- "rtp_payload_type_c2s",
- "rtp_payload_type_s2c",
- "rtp_pcap_path",
- "rtp_originator_dir",
- "stratum_cryptocurrency",
- "stratum_mining_pools",
- "stratum_mining_program",
- "stratum_mining_subscribe"
+ "app_extra_info"
],
"transmission": [
"sent_pkts",
@@ -2514,106 +2810,890 @@ "out_src_mac",
"in_dest_mac",
"out_dest_mac",
- "tunnels",
"dup_traffic_flag",
+ "tunnel_id_list",
"tunnel_endpoint_a_desc",
"tunnel_endpoint_b_desc"
]
},
"fields": {
- "encapsulation_type": {
+ "address_type": {
"data": [
{
- "code": "0",
- "value": "Ethernet"
+ "code": 4,
+ "value": "IPv4"
},
{
- "code": "8",
- "value": "PPP"
+ "code": 6,
+ "value": "IPv6"
+ }
+ ]
+ },
+ "mail_starttls_flag": {
+ "data": [
+ {
+ "code": 0,
+ "value": "QUERY"
},
{
- "code": "12",
- "value": "CiscoHDLC"
+ "code": 1,
+ "value": "RESPONSE"
+ }
+ ]
+ },
+ "dns_qr": {
+ "data": [
+ {
+ "code": 0,
+ "value": "False"
+ },
+ {
+ "code": 1,
+ "value": "True"
+ }
+ ]
+ },
+ "dns_qtype": {
+ "data": [
+ {
+ "code": 1,
+ "value": "A"
+ },
+ {
+ "code": 2,
+ "value": "NS"
+ },
+ {
+ "code": 3,
+ "value": "MD"
+ },
+ {
+ "code": 4,
+ "value": "MF"
+ },
+ {
+ "code": 5,
+ "value": "CNAME"
+ },
+ {
+ "code": 6,
+ "value": "SOA"
+ },
+ {
+ "code": 7,
+ "value": "MB"
+ },
+ {
+ "code": 8,
+ "value": "MG"
+ },
+ {
+ "code": 9,
+ "value": "MR"
+ },
+ {
+ "code": 10,
+ "value": "NULL"
+ },
+ {
+ "code": 11,
+ "value": "WKS"
+ },
+ {
+ "code": 12,
+ "value": "PTR"
+ },
+ {
+ "code": 13,
+ "value": "HINFO"
+ },
+ {
+ "code": 14,
+ "value": "MINFO"
+ },
+ {
+ "code": 15,
+ "value": "MX"
+ },
+ {
+ "code": 16,
+ "value": "TXT"
+ },
+ {
+ "code": 17,
+ "value": "RP"
+ },
+ {
+ "code": 18,
+ "value": "AFSDB"
+ },
+ {
+ "code": 19,
+ "value": "X25"
+ },
+ {
+ "code": 20,
+ "value": "ISDN"
+ },
+ {
+ "code": 21,
+ "value": "RT"
+ },
+ {
+ "code": 22,
+ "value": "NSAP"
+ },
+ {
+ "code": 23,
+ "value": "NSAP"
+ },
+ {
+ "code": 24,
+ "value": "SIG"
+ },
+ {
+ "code": 25,
+ "value": "KEY"
+ },
+ {
+ "code": 26,
+ "value": "PX"
+ },
+ {
+ "code": 27,
+ "value": "GPOS"
+ },
+ {
+ "code": 28,
+ "value": "AAAA"
+ },
+ {
+ "code": 29,
+ "value": "LOC"
+ },
+ {
+ "code": 30,
+ "value": "EID"
+ },
+ {
+ "code": 31,
+ "value": "NIMLOC"
+ },
+ {
+ "code": 32,
+ "value": "NB"
+ },
+ {
+ "code": 33,
+ "value": "SRV"
+ },
+ {
+ "code": 34,
+ "value": "ATMA"
+ },
+ {
+ "code": 35,
+ "value": "NAPTR"
+ },
+ {
+ "code": 36,
+ "value": "KX"
+ },
+ {
+ "code": 37,
+ "value": "CERT"
+ },
+ {
+ "code": 38,
+ "value": "A6"
+ },
+ {
+ "code": 39,
+ "value": "DNAME"
+ },
+ {
+ "code": 40,
+ "value": "SINK"
+ },
+ {
+ "code": 41,
+ "value": "OPT"
+ },
+ {
+ "code": 42,
+ "value": "APL"
+ },
+ {
+ "code": 43,
+ "value": "DS"
+ },
+ {
+ "code": 44,
+ "value": "SSHFP"
+ },
+ {
+ "code": 45,
+ "value": "IPSECKEY"
+ },
+ {
+ "code": 46,
+ "value": "RRSIG"
+ },
+ {
+ "code": 47,
+ "value": "NSEC"
+ },
+ {
+ "code": 48,
+ "value": "DNSKEY"
+ },
+ {
+ "code": 49,
+ "value": "DHCID"
+ },
+ {
+ "code": 50,
+ "value": "NSEC3"
+ },
+ {
+ "code": 51,
+ "value": "NSEC3PARAM"
+ },
+ {
+ "code": 52,
+ "value": "TLSA"
+ },
+ {
+ "code": 53,
+ "value": "SMIMEA"
+ },
+ {
+ "code": 55,
+ "value": "HIP"
+ },
+ {
+ "code": 59,
+ "value": "CDS"
+ },
+ {
+ "code": 60,
+ "value": "CDNSKEY"
+ },
+ {
+ "code": 61,
+ "value": "OPENPGPKEY"
+ },
+ {
+ "code": 62,
+ "value": "CSYNC"
+ },
+ {
+ "code": 63,
+ "value": "ZONEMD"
+ },
+ {
+ "code": 64,
+ "value": "SVCB"
+ },
+ {
+ "code": 65,
+ "value": "HTTPS"
+ },
+ {
+ "code": 99,
+ "value": "SPF"
+ },
+ {
+ "code": 100,
+ "value": "UINFO"
+ },
+ {
+ "code": 101,
+ "value": "UID"
+ },
+ {
+ "code": 102,
+ "value": "GID"
+ },
+ {
+ "code": 103,
+ "value": "UNSPEC"
+ },
+ {
+ "code": 108,
+ "value": "EUI48"
+ },
+ {
+ "code": 109,
+ "value": "EUI64"
+ },
+ {
+ "code": 249,
+ "value": "TKEY"
+ },
+ {
+ "code": 250,
+ "value": "TSIG"
+ },
+ {
+ "code": 251,
+ "value": "IXFR"
+ },
+ {
+ "code": 252,
+ "value": "AXFR"
+ },
+ {
+ "code": 253,
+ "value": "MAILB"
+ },
+ {
+ "code": 254,
+ "value": "MAILA"
+ },
+ {
+ "code": 255,
+ "value": "*"
+ },
+ {
+ "code": 256,
+ "value": "URI"
+ },
+ {
+ "code": 257,
+ "value": "CAA"
+ },
+ {
+ "code": 32768,
+ "value": "TA"
+ },
+ {
+ "code": 32769,
+ "value": "DLV"
+ },
+ {
+ "code": 65521,
+ "value": "INTEGRITY"
+ }
+ ]
+ },
+ "dns_rcode": {
+ "data": [
+ {
+ "code": 0,
+ "value": "NoError"
+ },
+ {
+ "code": 1,
+ "value": "FormErr"
+ },
+ {
+ "code": 2,
+ "value": "ServFail"
+ },
+ {
+ "code": 3,
+ "value": "NXDomain"
+ },
+ {
+ "code": 4,
+ "value": "NotImp"
+ },
+ {
+ "code": 5,
+ "value": "Refused"
+ },
+ {
+ "code": 6,
+ "value": "YXDomain"
+ },
+ {
+ "code": 7,
+ "value": "YXRRSet"
+ },
+ {
+ "code": 8,
+ "value": "NXRRSet"
+ },
+ {
+ "code": 9,
+ "value": "NotAuth"
+ },
+ {
+ "code": 10,
+ "value": "NotZone"
+ },
+ {
+ "code": 16,
+ "value": "BADSIG"
+ },
+ {
+ "code": 17,
+ "value": "BADKEY"
+ },
+ {
+ "code": 18,
+ "value": "BADTIME"
+ },
+ {
+ "code": 19,
+ "value": "BADMODE"
+ },
+ {
+ "code": 20,
+ "value": "BADNAME"
+ },
+ {
+ "code": 21,
+ "value": "BADALG"
+ }
+ ]
+ },
+ "dns_sub": {
+ "data": [
+ {
+ "code": 1,
+ "value": "DNS"
+ },
+ {
+ "code": 2,
+ "value": "DNSSEC"
+ }
+ ]
+ },
+ "direction": {
+ "data": [
+ {
+ "code": "Inbound",
+ "value": "Inbound"
+ },
+ {
+ "code": "Outbound",
+ "value": "Outbound"
+ }
+ ]
+ },
+ "security_action": {
+ "data": [
+ {
+ "code": "Deny",
+ "value": "Deny"
+ },
+ {
+ "code": "Allow",
+ "value": "Allow"
+ }
+ ]
+ },
+ "proxy_pinning_status": {
+ "data": [
+ {
+ "code": 0,
+ "value": "not pinning"
+ },
+ {
+ "code": 1,
+ "value": "pinning"
+ },
+ {
+ "code": 2,
+ "value": "maybe pinning"
+ }
+ ]
+ },
+ "proxy_intercept_status": {
+ "data": [
+ {
+ "code": 0,
+ "value": "passthrough"
+ },
+ {
+ "code": 1,
+ "value": "intercept"
+ },
+ {
+ "code": 2,
+ "value": "shutdown"
+ }
+ ]
+ },
+ "ip_protocol": {
+ "data": [
+ {
+ "code": "tcp",
+ "value": "tcp"
+ },
+ {
+ "code": "udp",
+ "value": "udp"
+ }
+ ]
+ },
+ "dns_opcode": {
+ "data": [
+ {
+ "code": 0,
+ "value": "QUERY"
+ },
+ {
+ "code": 1,
+ "value": "IQUERY"
+ },
+ {
+ "code": 2,
+ "value": "STATUS"
+ },
+ {
+ "code": 5,
+ "value": "UPDATE"
+ }
+ ]
+ },
+ "rtp_payload_type_c2s": {
+ "data": [
+ {
+ "code": 0,
+ "value": "PCMU"
+ },
+ {
+ "code": 1,
+ "value": "1016"
+ },
+ {
+ "code": 2,
+ "value": "G721"
+ },
+ {
+ "code": 3,
+ "value": "GSM"
+ },
+ {
+ "code": 4,
+ "value": "G723"
+ },
+ {
+ "code": 5,
+ "value": "DVI4_8000"
+ },
+ {
+ "code": 6,
+ "value": "DVI4_16000"
+ },
+ {
+ "code": 7,
+ "value": "LPC"
+ },
+ {
+ "code": 8,
+ "value": "PCMA"
+ },
+ {
+ "code": 9,
+ "value": "G722"
+ },
+ {
+ "code": 10,
+ "value": "L16_STEREO"
+ },
+ {
+ "code": 11,
+ "value": "L16_MONO"
+ },
+ {
+ "code": 12,
+ "value": "QCELP"
+ },
+ {
+ "code": 13,
+ "value": "CN"
+ },
+ {
+ "code": 14,
+ "value": "MPA"
+ },
+ {
+ "code": 15,
+ "value": "G728"
+ },
+ {
+ "code": 16,
+ "value": "DVI4_11025"
+ },
+ {
+ "code": 17,
+ "value": "DVI4_22050"
+ },
+ {
+ "code": 18,
+ "value": "G729"
+ },
+ {
+ "code": 19,
+ "value": "CN_OLD"
+ },
+ {
+ "code": 25,
+ "value": "CELB"
+ },
+ {
+ "code": 26,
+ "value": "JPEG"
+ },
+ {
+ "code": 28,
+ "value": "NV"
+ },
+ {
+ "code": 31,
+ "value": "H261"
+ },
+ {
+ "code": 32,
+ "value": "MPV"
+ },
+ {
+ "code": 33,
+ "value": "MP2T"
+ },
+ {
+ "code": 34,
+ "value": "H263"
+ }
+ ]
+ },
+ "rtp_payload_type_s2c": {
+ "data": [
+ {
+ "code": 0,
+ "value": "PCMU"
+ },
+ {
+ "code": 1,
+ "value": "1016"
+ },
+ {
+ "code": 2,
+ "value": "G721"
+ },
+ {
+ "code": 3,
+ "value": "GSM"
+ },
+ {
+ "code": 4,
+ "value": "G723"
+ },
+ {
+ "code": 5,
+ "value": "DVI4_8000"
+ },
+ {
+ "code": 6,
+ "value": "DVI4_16000"
+ },
+ {
+ "code": 7,
+ "value": "LPC"
+ },
+ {
+ "code": 8,
+ "value": "PCMA"
+ },
+ {
+ "code": 9,
+ "value": "G722"
+ },
+ {
+ "code": 10,
+ "value": "L16_STEREO"
+ },
+ {
+ "code": 11,
+ "value": "L16_MONO"
+ },
+ {
+ "code": 12,
+ "value": "QCELP"
+ },
+ {
+ "code": 13,
+ "value": "CN"
+ },
+ {
+ "code": 14,
+ "value": "MPA"
+ },
+ {
+ "code": 15,
+ "value": "G728"
+ },
+ {
+ "code": 16,
+ "value": "DVI4_11025"
+ },
+ {
+ "code": 17,
+ "value": "DVI4_22050"
+ },
+ {
+ "code": 18,
+ "value": "G729"
+ },
+ {
+ "code": 19,
+ "value": "CN_OLD"
+ },
+ {
+ "code": 25,
+ "value": "CELB"
+ },
+ {
+ "code": 26,
+ "value": "JPEG"
+ },
+ {
+ "code": 28,
+ "value": "NV"
+ },
+ {
+ "code": 31,
+ "value": "H261"
+ },
+ {
+ "code": 32,
+ "value": "MPV"
+ },
+ {
+ "code": 33,
+ "value": "MP2T"
+ },
+ {
+ "code": 34,
+ "value": "H263"
+ }
+ ]
+ },
+ "rtp_originator_dir": {
+ "data": [
+ {
+ "code": 0,
+ "value": "unknown"
+ },
+ {
+ "code": 1,
+ "value": "c2s"
+ },
+ {
+ "code": 2,
+ "value": "s2c"
}
]
},
"dup_traffic_flag": {
"data": [
{
- "code": "0",
- "value": "No"
+ "code": 0,
+ "value": "False"
+ },
+ {
+ "code": 1,
+ "value": "True"
+ }
+ ]
+ },
+ "ssl_esni_flag": {
+ "data": [
+ {
+ "code": 0,
+ "value": "False"
+ },
+ {
+ "code": 1,
+ "value": "True"
+ }
+ ]
+ },
+ "ssl_ech_flag": {
+ "data": [
+ {
+ "code": 0,
+ "value": "False"
},
{
- "code": "1",
- "value": "Yes"
+ "code": 1,
+ "value": "True"
}
]
},
"flags": {
"data": [
{
- "code": "1",
- "value": "Asymmetric"
+ "code": 1,
+ "value": "Asymmetric(Deprecated)"
},
{
- "code": "2",
+ "code": 2,
"value": "Bulky"
},
{
- "code": "4",
+ "code": 4,
"value": "CBR Streaming"
},
{
- "code": "8",
+ "code": 8,
"value": "Client is Local"
},
{
- "code": "16",
+ "code": 16,
"value": "Server is Local"
},
{
- "code": "32",
+ "code": 32,
"value": "Download"
},
{
- "code": "64",
+ "code": 64,
"value": "Interactive"
},
{
- "code": "128",
+ "code": 128,
"value": "Inbound"
},
{
- "code": "256",
+ "code": 256,
"value": "Outbound"
},
{
- "code": "512",
+ "code": 512,
"value": "Pseudo Unidirectional"
},
{
- "code": "1024",
+ "code": 1024,
"value": "Streaming"
},
{
- "code": "2048",
+ "code": 2048,
"value": "Unidirectional"
},
{
- "code": "4096",
+ "code": 4096,
"value": "Random looking"
},
{
- "code": "8192",
+ "code": 8192,
"value": "C2S"
},
{
- "code": "16384",
+ "code": 16384,
"value": "S2C"
},
{
- "code": "32768",
+ "code": 32768,
"value": "Bidirectional"
+ },
+ {
+ "code": 65536,
+ "value": "Tunneling"
}
]
}
diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/public_schema_info_2311.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/public_schema_info_2311.json deleted file mode 100644 index 2928baad..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/public_schema_info_2311.json +++ /dev/null @@ -1,3196 +0,0 @@ -{
- "functions": {
- "aggregation": [
- {
- "name": "COUNT",
- "label": "COUNT",
- "function": "count(expr)"
- },
- {
- "name": "COUNT_DISTINCT",
- "label": "COUNT_DISTINCT",
- "function": "count(distinct expr)"
- },
- {
- "name": "AVG",
- "label": "AVG",
- "function": "avg(expr)"
- },
- {
- "name": "SUM",
- "label": "SUM",
- "function": "sum(expr)"
- },
- {
- "name": "MAX",
- "label": "MAX",
- "function": "max(expr)"
- },
- {
- "name": "MIN",
- "label": "MIN",
- "function": "min(expr)"
- },
- {
- "name": "MEDIAN",
- "label": "MEDIAN",
- "function": "MEDIAN(expr)"
- },
- {
- "name": "QUANTILE",
- "label": "QUANTILE",
- "function": "QUANTILE(expr,level)"
- },
- {
- "name": "TIME_FLOOR_WITH_FILL",
- "label": "TIME_FLOOR_WITH_FILL",
- "function": "TIME_FLOOR_WITH_FILL(expr,period,fill)"
- }
- ],
- "date": [
- {
- "name": "UNIX_TIMESTAMP",
- "label": "UNIX_TIMESTAMP",
- "function": "UNIX_TIMESTAMP(expr)"
- },
- {
- "name": "FROM_UNIXTIME",
- "label": "FROM_UNIXTIME",
- "function": "FROM_UNIXTIME(expr)"
- },
- {
- "name": "DATE_FORMAT",
- "label": "DATE_FORMAT",
- "function": "DATE_FORMAT(expr,format)"
- },
- {
- "name": "CONVERT_TZ",
- "label": "CONVERT_TZ",
- "function": "CONVERT_TZ(expr, from_tz, to_tz)"
- }
- ],
- "operator": [
- {
- "name": "=",
- "label": "=",
- "function": "expr = value"
- },
- {
- "name": "!=",
- "label": "!=",
- "function": "expr != value"
- },
- {
- "name": ">",
- "label": ">",
- "function": "expr > value"
- },
- {
- "name": "<",
- "label": "<",
- "function": "expr < value"
- },
- {
- "name": ">=",
- "label": ">=",
- "function": "expr >= value"
- },
- {
- "name": "<=",
- "label": "<=",
- "function": "expr <= value"
- },
- {
- "name": "has",
- "label": "HAS",
- "function": "has(expr, value)"
- },
- {
- "name": "in",
- "label": "IN",
- "function": "expr in (values)"
- },
- {
- "name": "not in",
- "label": "NOT IN",
- "function": "expr not in (values)"
- },
- {
- "name": "like",
- "label": "LIKE",
- "function": "expr like value"
- },
- {
- "name": "not like",
- "label": "NOT LIKE",
- "function": "expr not like value"
- },
- {
- "name": "notEmpty",
- "label": "NOT EMPTY",
- "function": "notEmpty(expr)"
- },
- {
- "name": "empty",
- "label": "EMPTY",
- "function": "empty(expr)"
- },
- {
- "name": "bitAnd",
- "label": "Bitwise AND",
- "function": "bitAnd(expr, value)=value"
- }
- ],
- "sketch": [
- {
- "name": "APPROX_COUNT_DISTINCT_HLLD",
- "label": "APPROX_COUNT_DISTINCT_HLLD",
- "function": "APPROX_COUNT_DISTINCT_HLLD(expr, precision,round)"
- },
- {
- "name": "HLLD",
- "label": "HLLD",
- "function": "HLLD(expr, precision)"
- },
- {
- "name": "APPROX_QUANTILE_HDR",
- "label": "APPROX_QUANTILE_HDR",
- "function": "APPROX_QUANTILE_HDR(expr,probability,numberOfSignificantValueDigits)"
- },
- {
- "name": "HDR_HISTOGRAM",
- "label": "HDR_HISTOGRAM",
- "function": "HDR_HISTOGRAM(expr, numberOfSignificantValueDigits)"
- },
- {
- "name": "HDR_GET_QUANTILES",
- "label": "HDR_GET_QUANTILES",
- "function": "HDR_GET_QUANTILES(expr,probability0, probability1, ...)"
- },
- {
- "name": "HDR_GET_PERCENTILES",
- "label": "HDR_GET_PERCENTILES",
- "function": "HDR_GET_QUANTILES(expr,percentileTicksPerHalfDistance)"
- }
- ]
- },
- "units": {
- "bytes": {
- "label": "Bytes",
- "abbreviation": "B"
- },
- "bits": {
- "label": "Bits",
- "abbreviation": "b"
- },
- "packets": {
- "label": "Packets",
- "abbreviation": "Pkts"
- },
- "flows": {
- "label": "Flows",
- "abbreviation": "Flows"
- },
- "sessions": {
- "label": "Sessions",
- "abbreviation": "Sessions"
- },
- "hit_count": {
- "label": "Hit Count",
- "abbreviation": "Hit Count"
- },
- "bps": {
- "label": "bits/s",
- "abbreviation": "bps",
- "desc": "bit per second"
- },
- "Bps": {
- "label": "Bytes/s",
- "abbreviation": "Bps",
- "desc": "Byte per second"
- },
- "pps": {
- "label": "Packets/s",
- "abbreviation": "pps",
- "desc": "Packet per second"
- },
- "fps": {
- "label": "Flows/s",
- "abbreviation": "fps",
- "desc": "Flow per second"
- },
- "sps": {
- "label": "Sessions/s",
- "abbreviation": "Sessions/s",
- "desc": "Session per second"
- },
- "cps": {
- "label": "Count/s",
- "abbreviation": "Count/s",
- "desc": "count per second"
- },
- "ms": {
- "label": "millisecond",
- "abbreviation": "ms"
- },
- "s": {
- "label": "second",
- "abbreviation": "s"
- },
- "unique_ip": {
- "label": "Unique IPs",
- "abbreviation": "Unique IPs"
- },
- "percentage": {
- "label": "%",
- "abbreviation": "%"
- }
- },
- "radius_onff_log": {
- "index_key": {
- "log_id": [
- "account",
- "event_timestamp",
- "vsys_id"
- ],
- "vsys_id": [
- "vsys_id",
- "account",
- "event_timestamp"
- ]
- }
- },
- "session_record": {
- "index_key": {
- "log_id": [
- "common_log_id",
- "common_data_center",
- "common_recv_time",
- "common_vsys_id"
- ],
- "vsys_id": [
- "common_vsys_id",
- "common_data_center",
- "common_device_group",
- "common_recv_time",
- "common_log_id"
- ]
- }
- },
- "transaction_record": {
- "index_key": {
- "log_id": [
- "common_stream_trace_id",
- "common_data_center",
- "common_recv_time",
- "common_vsys_id"
- ],
- "vsys_id": [
- "common_vsys_id",
- "common_stream_trace_id",
- "common_data_center",
- "common_recv_time"
- ]
- }
- },
- "radius_record": {
- "index_key": {
- "log_id": [
- "common_log_id",
- "common_data_center",
- "common_recv_time",
- "common_vsys_id"
- ],
- "vsys_id": [
- "common_vsys_id",
- "common_data_center",
- "common_recv_time",
- "common_log_id"
- ]
- }
- },
- "voip_record": {
- "index_key": {
- "log_id": [
- "common_log_id",
- "common_data_center",
- "common_recv_time",
- "common_vsys_id"
- ],
- "vsys_id": [
- "common_vsys_id",
- "common_data_center",
- "common_recv_time",
- "common_log_id"
- ]
- }
- },
- "gtpc_record": {
- "index_key": {
- "log_id": [
- "common_log_id",
- "common_data_center",
- "common_recv_time",
- "common_vsys_id"
- ],
- "vsys_id": [
- "common_vsys_id",
- "common_data_center",
- "common_recv_time",
- "common_log_id"
- ]
- }
- },
- "security_event": {
- "index_key": {
- "log_id": [
- "common_log_id",
- "common_policy_id",
- "common_recv_time",
- "common_vsys_id"
- ],
- "vsys_id": [
- "common_vsys_id",
- "common_action",
- "common_policy_id",
- "common_recv_time",
- "common_log_id"
- ]
- }
- },
- "active_defence_event": {
- "index_key": {
- "log_id": [
- "common_log_id",
- "common_policy_id",
- "common_recv_time"
- ],
- "vsys_id": [
- "common_log_id",
- "common_policy_id",
- "common_recv_time"
- ]
- }
- },
- "sys_packet_capture_event": {
- "index_key": {
- "log_id": [
- "common_log_id",
- "common_policy_id",
- "common_recv_time",
- "common_vsys_id"
- ],
- "vsys_id": [
- "common_vsys_id",
- "common_policy_id",
- "common_recv_time",
- "common_log_id"
- ]
- }
- },
- "schema_query": {
- "references": {
- "aggregation": [
- {
- "type": "int",
- "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,QUANTILE"
- },
- {
- "type": "long",
- "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,QUANTILE"
- },
- {
- "type": "float",
- "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,QUANTILE"
- },
- {
- "type": "double",
- "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,QUANTILE"
- },
- {
- "type": "string",
- "functions": "COUNT,COUNT_DISTINCT"
- },
- {
- "type": "date",
- "functions": "COUNT,COUNT_DISTINCT,MAX,MIN"
- },
- {
- "type": "datetime",
- "functions": "COUNT,COUNT_DISTINCT,MAX,MIN"
- },
- {
- "type": "timestamp",
- "functions": "COUNT,COUNT_DISTINCT,MAX,MIN"
- },
- {
- "type": "unix_timestamp",
- "functions": "COUNT,COUNT_DISTINCT,MAX,MIN"
- },
- {
- "type": "array",
- "functions": "COUNT,COUNT_DISTINCT"
- },
- {
- "type": "bit",
- "functions": "COUNT,COUNT_DISTINCT"
- }
- ],
- "operator": [
- {
- "type": "int",
- "functions": "=,!=,>,<,>=,<=,in,not in"
- },
- {
- "type": "long",
- "functions": "=,!=,>,<,>=,<=,in,not in"
- },
- {
- "type": "float",
- "functions": "=,!=,>,<,>=,<="
- },
- {
- "type": "double",
- "functions": "=,!=,>,<,>=,<="
- },
- {
- "type": "string",
- "functions": "=,!=,in,not in,like,not like,notEmpty,empty"
- },
- {
- "type": "date",
- "functions": "=,!=,>,<,>=,<="
- },
- {
- "type": "dateTime",
- "functions": "=,!=,>,<,>=,<="
- },
- {
- "type": "timestamp",
- "functions": "=,!=,>,<,>=,<="
- },
- {
- "type": "unix_timestamp",
- "functions": "=,!=,>,<,>=,<="
- },
- {
- "type": "array",
- "functions": "has,notEmpty,empty"
- },
- {
- "type": "bit",
- "functions": "=,!=,bitAnd"
- }
- ]
- }
- },
- "schema_type": {
- "BASE": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_service_category",
- "common_l7_protocol",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "common_server_ip",
- "common_server_port"
- ]
- },
- "HTTP": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_service_category",
- "common_l7_protocol",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "http_url",
- "http_host",
- "http_domain",
- "http_request_line",
- "http_response_line",
- "http_request_header",
- "http_response_header",
- "http_request_content",
- "http_request_content_length",
- "http_request_content_type",
- "http_response_content",
- "http_response_content_length",
- "http_response_content_type",
- "http_request_body",
- "http_response_body",
- "http_proxy_flag",
- "http_sequence",
- "http_snapshot",
- "http_cookie",
- "http_referer",
- "http_user_agent",
- "http_set_cookie",
- "http_version",
- "http_status_code",
- "http_response_latency_ms",
- "http_session_duration_ms",
- "http_action_file_size"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "http_url",
- "common_server_port",
- "common_sub_action"
- ]
- },
- "MAIL": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "mail_protocol_type",
- "mail_account",
- "mail_from_cmd",
- "mail_to_cmd",
- "mail_from",
- "mail_to",
- "mail_cc",
- "mail_bcc",
- "mail_subject",
- "mail_subject_charset",
- "mail_content",
- "mail_content_charset",
- "mail_attachment_name",
- "mail_attachment_name_charset",
- "mail_attachment_content",
- "mail_eml_file",
- "mail_snapshot"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "mail_from",
- "mail_to",
- "mail_subject"
- ]
- },
- "DNS": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "dns_message_id",
- "dns_qr",
- "dns_opcode",
- "dns_aa",
- "dns_tc",
- "dns_rd",
- "dns_ra",
- "dns_rcode",
- "dns_qdcount",
- "dns_ancount",
- "dns_nscount",
- "dns_arcount",
- "dns_qname",
- "dns_qtype",
- "dns_qclass",
- "dns_cname",
- "dns_sub",
- "dns_rr",
- "dns_response_latency_ms"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_client_ip",
- "dns_qr",
- "dns_qname",
- "dns_qtype"
- ]
- },
- "SSL": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "ssl_version",
- "ssl_sni",
- "ssl_san",
- "ssl_cn",
- "ssl_pinningst",
- "ssl_intercept_state",
- "ssl_passthrough_reason",
- "ssl_server_side_latency",
- "ssl_client_side_latency",
- "ssl_server_side_version",
- "ssl_client_side_version",
- "ssl_cert_verify",
- "ssl_error",
- "ssl_con_latency_ms",
- "ssl_ja3_fingerprint",
- "ssl_ja3_hash",
- "ssl_ja3s_fingerprint",
- "ssl_ja3s_hash",
- "ssl_cert_issuer",
- "ssl_cert_subject",
- "ssl_esni_flag",
- "ssl_ech_flag"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "ssl_sni",
- "common_server_ip",
- "common_server_port"
- ]
- },
- "DTLS": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "dtls_cookie",
- "dtls_version",
- "dtls_sni",
- "dtls_san",
- "dtls_cn",
- "dtls_con_latency_ms",
- "dtls_ja3_fingerprint",
- "dtls_ja3_hash",
- "dtls_cert_issuer",
- "dtls_cert_subject"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "dtls_sni",
- "common_server_ip",
- "common_server_port"
- ]
- },
- "QUIC": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "quic_version",
- "quic_sni",
- "quic_user_agent"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "quic_sni",
- "common_server_ip",
- "common_server_port"
- ]
- },
- "FTP": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "ftp_account",
- "ftp_url",
- "ftp_content",
- "ftp_link_type"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "ftp_url",
- "common_server_ip",
- "common_server_port"
- ]
- },
- "BGP": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "bgp_type",
- "bgp_as_num",
- "bgp_route"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "bgp_type",
- "bgp_as_num",
- "common_server_ip",
- "common_server_port"
- ]
- },
- "SIP": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "sip_call_id",
- "sip_originator_description",
- "sip_responder_description",
- "sip_user_agent",
- "sip_server",
- "sip_originator_sdp_connect_ip",
- "sip_originator_sdp_media_port",
- "sip_originator_sdp_media_type",
- "sip_originator_sdp_content",
- "sip_responder_sdp_connect_ip",
- "sip_responder_sdp_media_port",
- "sip_responder_sdp_media_type",
- "sip_responder_sdp_content",
- "sip_duration_s",
- "sip_bye"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_subscriber_id",
- "common_client_ip",
- "sip_originator_description",
- "sip_responder_description",
- "sip_call_id",
- "common_server_ip",
- "common_server_port"
- ]
- },
- "RTP": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "rtp_payload_type_c2s",
- "rtp_payload_type_s2c",
- "rtp_pcap_path",
- "rtp_originator_dir"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_subscriber_id",
- "common_client_ip",
- "common_server_ip",
- "common_server_port",
- "rtp_pcap_path",
- "rtp_originator_dir"
- ]
- },
- "APP": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "app_extra_info"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "common_app_id",
- "common_app_label",
- "app_extra_info",
- "common_server_ip",
- "common_server_port"
- ]
- },
- "DoH": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "doh_url",
- "doh_host",
- "doh_request_line",
- "doh_response_line",
- "doh_cookie",
- "doh_referer",
- "doh_user_agent",
- "doh_content_length",
- "doh_content_type",
- "doh_set_cookie",
- "doh_version",
- "doh_message_id",
- "doh_qr",
- "doh_opcode",
- "doh_aa",
- "doh_tc",
- "doh_rd",
- "doh_ra",
- "doh_rcode",
- "doh_qdcount",
- "doh_ancount",
- "doh_nscount",
- "doh_arcount",
- "doh_qname",
- "doh_qtype",
- "doh_qclass",
- "doh_cname",
- "doh_sub",
- "doh_rr"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_client_ip",
- "doh_url",
- "doh_qname",
- "common_server_port"
- ]
- },
- "VoIP": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "sip_call_id",
- "sip_originator_description",
- "sip_responder_description",
- "sip_user_agent",
- "sip_server",
- "sip_originator_sdp_connect_ip",
- "sip_originator_sdp_media_port",
- "sip_originator_sdp_media_type",
- "sip_originator_sdp_content",
- "sip_responder_sdp_connect_ip",
- "sip_responder_sdp_media_port",
- "sip_responder_sdp_media_type",
- "sip_responder_sdp_content",
- "sip_duration_s",
- "sip_bye",
- "rtp_payload_type_c2s",
- "rtp_payload_type_s2c",
- "rtp_pcap_path",
- "rtp_originator_dir"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_subscriber_id",
- "common_client_ip",
- "sip_originator_description",
- "sip_responder_description",
- "sip_call_id",
- "common_server_ip",
- "common_server_port",
- "rtp_pcap_path",
- "rtp_originator_dir"
- ]
- },
- "SSH": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "ssh_version",
- "ssh_auth_success",
- "ssh_client_version",
- "ssh_server_version",
- "ssh_cipher_alg",
- "ssh_mac_alg",
- "ssh_compression_alg",
- "ssh_kex_alg",
- "ssh_host_key_alg",
- "ssh_host_key",
- "ssh_hassh"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_client_ip",
- "common_server_ip",
- "common_server_port",
- "ssh_auth_success"
- ]
- },
- "RADIUS": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "radius_packet_type",
- "radius_nas_ip",
- "radius_framed_ip",
- "radius_account",
- "radius_session_timeout",
- "radius_idle_timeout",
- "radius_acct_status_type",
- "radius_acct_terminate_cause",
- "radius_event_timestamp",
- "radius_nas_port",
- "radius_service_type",
- "radius_framed_protocol",
- "radius_callback_number",
- "radius_callback_id",
- "radius_termination_action",
- "radius_called_station_id",
- "radius_calling_station_id",
- "radius_acct_delay_time",
- "radius_acct_session_id",
- "radius_acct_multi_session_id",
- "radius_acct_input_octets",
- "radius_acct_output_octets",
- "radius_acct_input_packets",
- "radius_acct_output_packets",
- "radius_acct_session_time",
- "radius_acct_link_count",
- "radius_acct_interim_interval",
- "radius_acct_authentic"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_subscriber_id",
- "radius_nas_ip",
- "radius_framed_ip",
- "radius_acct_status_type"
- ]
- },
- "Stratum": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "stratum_cryptocurrency",
- "stratum_mining_pools",
- "stratum_mining_program"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_subscriber_id",
- "stratum_cryptocurrency",
- "stratum_mining_pools",
- "stratum_mining_program"
- ]
- },
- "RDP": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_l7_protocol",
- "common_service_category",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "rdp_cookie",
- "rdp_security_protocol",
- "rdp_client_channels",
- "rdp_keyboard_layout",
- "rdp_client_version",
- "rdp_client_name",
- "rdp_client_product_id",
- "rdp_desktop_width",
- "rdp_desktop_height",
- "rdp_requested_color_depth",
- "rdp_certificate_type",
- "rdp_certificate_count",
- "rdp_certificate_permanent",
- "rdp_encryption_level",
- "rdp_encryption_method"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "common_subscriber_id",
- "rdp_client_version",
- "rdp_client_name"
- ]
- },
- "GTP-C": {
- "columns": [
- "common_recv_time",
- "common_log_id",
- "common_policy_id",
- "common_subscriber_id",
- "common_imei",
- "common_imsi",
- "common_phone_number",
- "common_in_src_mac",
- "common_out_src_mac",
- "common_client_ip",
- "common_client_port",
- "common_client_os_name",
- "common_internal_ip",
- "common_l4_protocol",
- "common_address_type",
- "common_server_ip",
- "common_server_port",
- "common_server_os_name",
- "common_external_ip",
- "common_action",
- "common_direction",
- "common_entrance_id",
- "common_sled_ip",
- "common_client_location",
- "common_client_asn",
- "common_server_location",
- "common_server_asn",
- "common_server_fqdn",
- "common_server_domain",
- "common_in_dest_mac",
- "common_out_dest_mac",
- "common_sessions",
- "common_c2s_pkt_num",
- "common_s2c_pkt_num",
- "common_c2s_byte_num",
- "common_s2c_byte_num",
- "common_c2s_pkt_diff",
- "common_s2c_pkt_diff",
- "common_c2s_byte_diff",
- "common_s2c_byte_diff",
- "common_service",
- "common_schema_type",
- "common_vsys_id",
- "common_t_vsys_id",
- "common_flags",
- "common_flags_identify_info",
- "common_user_tags",
- "common_sub_action",
- "common_user_region",
- "common_shaping_rule_ids",
- "common_device_id",
- "common_out_link_id",
- "common_in_link_id",
- "common_isp",
- "common_device_tag",
- "common_data_center",
- "common_device_group",
- "common_app_behavior",
- "common_encapsulation",
- "common_app_label",
- "common_tunnels",
- "common_protocol_label",
- "common_app_id",
- "common_app_full_path",
- "common_userdefine_app_name",
- "common_app_identify_info",
- "common_app_surrogate_id",
- "common_service_category",
- "common_l7_protocol",
- "common_start_time",
- "common_end_time",
- "common_start_timestamp_ms",
- "common_end_timestamp_ms",
- "common_establish_latency_ms",
- "common_con_duration_ms",
- "common_stream_dir",
- "common_address_list",
- "common_has_dup_traffic",
- "common_stream_error",
- "common_stream_trace_id",
- "common_link_info_c2s",
- "common_link_info_s2c",
- "common_packet_capture_file",
- "common_tunnel_endpoint_a_desc",
- "common_tunnel_endpoint_b_desc",
- "common_c2s_ipfrag_num",
- "common_s2c_ipfrag_num",
- "common_c2s_tcp_lostlen",
- "common_s2c_tcp_lostlen",
- "common_c2s_tcp_unorder_num",
- "common_s2c_tcp_unorder_num",
- "common_c2s_pkt_retrans",
- "common_s2c_pkt_retrans",
- "common_c2s_byte_retrans",
- "common_s2c_byte_retrans",
- "common_tcp_client_isn",
- "common_tcp_server_isn",
- "common_first_ttl",
- "common_processing_time",
- "common_ingestion_time",
- "common_mirrored_pkts",
- "common_mirrored_bytes",
- "gtp_version",
- "gtp_apn",
- "gtp_imei",
- "gtp_imsi",
- "gtp_phone_number",
- "gtp_end_user_ipv4",
- "gtp_end_user_ipv6",
- "gtp_uplink_teid",
- "gtp_downlink_teid",
- "gtp_msg_type"
- ],
- "default_columns": [
- "common_recv_time",
- "common_log_id",
- "gtp_version",
- "gtp_msg_type",
- "gtp_imsi",
- "gtp_imei",
- "gtp_phone_number",
- "common_client_ip",
- "common_server_ip"
- ]
- }
- },
- "tunnel_type": {
- "GTP": [
- {
- "name": "gtp_endpoint_a_ip",
- "label": "Endpoint A IP",
- "type": "string"
- },
- {
- "name": "gtp_endpoint_b_ip",
- "label": "Endpoint B IP",
- "type": "string"
- },
- {
- "name": "gtp_endpoint_a_port",
- "label": "Endpoint A Port",
- "type": "int"
- },
- {
- "name": "gtp_endpoint_b_port",
- "label": "Endpoint B Port",
- "type": "int"
- },
- {
- "name": "gtp_endpoint_a2b_teid",
- "label": "Endpoint A2B TEID",
- "type": "long"
- },
- {
- "name": "gtp_endpoint_b2a_teid",
- "label": "Endpoint B2A TEID",
- "type": "long"
- }
- ],
- "MPLS": [
- {
- "name": "mpls_c2s_direction_label",
- "label": "Multiprotocol Label (c2s)",
- "type": {
- "type": "array",
- "items": "int",
- "logicalType": "array"
- }
- },
- {
- "name": "mpls_s2c_direction_label",
- "label": "Multiprotocol Label (s2c)",
- "type": {
- "type": "array",
- "items": "int",
- "logicalType": "array"
- }
- }
- ],
- "VLAN": [
- {
- "name": "vlan_c2s_direction_id",
- "label": "VLAN Direction (c2s)",
- "type": {
- "type": "array",
- "items": "int",
- "logicalType": "array"
- }
- },
- {
- "name": "vlan_s2c_direction_id",
- "label": "VLAN Direction (s2c)",
- "type": {
- "type": "array",
- "items": "int",
- "logicalType": "array"
- }
- }
- ],
- "ETHERNET": [
- {
- "name": "source_mac",
- "label": "Source MAC",
- "type": "string"
- },
- {
- "name": "destination_mac",
- "label": "Destination MAC",
- "type": "string"
- }
- ],
- "MULTIPATH_ETHERNET": [
- {
- "name": "c2s_source_mac",
- "label": "Source MAC (c2s)",
- "type": "string"
- },
- {
- "name": "c2s_destination_mac",
- "label": "Destination MAC (c2s)",
- "type": "string"
- },
- {
- "name": "s2c_source_mac",
- "label": "Source MAC (s2c)",
- "type": "string"
- },
- {
- "name": "s2c_destination_mac",
- "label": "Destination MAC (s2c)",
- "type": "string"
- }
- ],
- "L2TP": [
- {
- "name": "l2tp_version",
- "label": "Version",
- "type": "string"
- },
- {
- "name": "l2tp_lac2lns_tunnel_id",
- "label": "LAC2LNS Tunnel ID",
- "type": "int"
- },
- {
- "name": "l2tp_lns2lac_tunnel_id",
- "label": "LNS2LAC Tunnel ID",
- "type": "int"
- },
- {
- "name": "l2tp_lac2lns_session_id",
- "label": "LAC2LNS Session ID",
- "type": "int"
- },
- {
- "name": "l2tp_lns2lac_session_id",
- "label": "LNS2LAC Session ID",
- "type": "int"
- },
- {
- "name": "l2tp_access_concentrator_ip",
- "label": "Access Concentrator IP",
- "type": "string"
- },
- {
- "name": "l2tp_access_concentrator_port",
- "label": "Access Concentrator Port",
- "type": "int"
- },
- {
- "name": "l2tp_network_server_ip",
- "label": "Network Server IP",
- "type": "string"
- },
- {
- "name": "l2tp_network_server_port",
- "label": "Network Server Port",
- "type": "int"
- }
- ],
- "PPTP": [
- {
- "name": "pptp_uplink_tunnel_id",
- "label": "UpLink Tunnel ID",
- "type": "int"
- },
- {
- "name": "pptp_downlink_tunnel_id",
- "label": "Down Tunnel ID",
- "type": "int"
- }
- ]
- },
- "fields": {
- "common_encapsulation": {
- "data": [
- {
- "code": "0",
- "value": "Ethernet"
- },
- {
- "code": "8",
- "value": "PPP"
- },
- {
- "code": "12",
- "value": "CiscoHDLC"
- }
- ]
- },
- "common_has_dup_traffic": {
- "data": [
- {
- "code": "0",
- "value": "No"
- },
- {
- "code": "1",
- "value": "Yes"
- }
- ]
- },
- "common_flags": {
- "data": [
- {
- "code": "1",
- "value": "Asymmetric"
- },
- {
- "code": "2",
- "value": "Bulky"
- },
- {
- "code": "4",
- "value": "CBR Streaming"
- },
- {
- "code": "8",
- "value": "Client is Local"
- },
- {
- "code": "16",
- "value": "Server is Local"
- },
- {
- "code": "32",
- "value": "Download"
- },
- {
- "code": "64",
- "value": "Interactive"
- },
- {
- "code": "128",
- "value": "Inbound"
- },
- {
- "code": "256",
- "value": "Outbound"
- },
- {
- "code": "512",
- "value": "Pseudo Unidirectional"
- },
- {
- "code": "1024",
- "value": "Streaming"
- },
- {
- "code": "2048",
- "value": "Unidirectional"
- },
- {
- "code": "4096",
- "value": "Random looking"
- },
- {
- "code": "8192",
- "value": "C2S"
- },
- {
- "code": "16384",
- "value": "S2C"
- },
- {
- "code": "32768",
- "value": "Bidirectional"
- }
- ]
- }
- }
-}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/radius_onff_log.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/radius_onff_log.json deleted file mode 100644 index 5bf3fdd4..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/radius_onff_log.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "type": "record", - "name": "radius_onff_log", - "namespace": "tsg_galaxy_v3", - "doc": { - "partition_key": "event_timestamp", - "index_key": { - "$ref": "public_schema_info_2311.json#/radius_onff_log/index_key" - } - }, - "fields": [ - { - "name": "event_timestamp", - "label": "Event Time", - "doc": { - "visibility": "enabled" - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "account", - "label": "Account", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "vsys_id", - "label": "Vsys ID", - "doc": { - "visibility": "enabled" - }, - "type": "int", - "default": 1 - }, - { - "name": "framed_ip", - "label": "Framed IP", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "acct_session_id", - "label": "Acct Session ID", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "acct_status_type", - "label": "Acct Status Type", - "doc": { - "visibility": "enabled" - }, - "type": "int" - }, - { - "name": "acct_session_time", - "label": "Acct Session Time", - "doc": { - "visibility": "enabled" - }, - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/radius_record.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/radius_record.json deleted file mode 100644 index 2b30a55e..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/radius_record.json +++ /dev/null @@ -1,2052 +0,0 @@ -{ - "type": "record", - "name": "radius_record", - "namespace": "tsg_galaxy_v3", - "doc": { - "primary_key": "common_log_id", - "partition_key": "common_recv_time", - "ttl": null, - "default_ttl": 2592000, - "index_key": { - "$ref": "public_schema_info_2311.json#/radius_record/index_key" - }, - "functions": { - "$ref": "public_schema_info_2311.json#/functions" - }, - "schema_query": { - "dimensions": [ - "common_stream_trace_id", - "common_stream_dir", - "common_start_time", - "common_end_time", - "common_start_timestamp_ms", - "common_end_timestamp_ms", - "common_processing_time", - "common_ingestion_time", - "common_device_id", - "common_data_center", - "common_sled_ip", - "common_device_group", - "common_address_type", - "common_vsys_id", - "common_client_ip", - "common_internal_ip", - "common_client_port", - "common_client_os_name", - "common_client_location", - "common_client_asn", - "common_server_ip", - "common_external_ip", - "common_server_port", - "common_server_os_name", - "common_server_location", - "common_server_asn", - "common_server_fqdn", - "common_server_domain", - "common_in_dest_mac", - "common_out_dest_mac", - "common_l4_protocol", - "common_tunnel_endpoint_a_desc", - "common_tunnel_endpoint_b_desc", - "common_subscriber_id", - "common_in_src_mac", - "common_out_src_mac", - "radius_nas_ip", - "radius_framed_ip", - "radius_packet_type", - "radius_account", - "radius_acct_status_type", - "radius_acct_terminate_cause", - "radius_nas_port", - "radius_called_station_id", - "radius_calling_station_id", - "radius_acct_session_id", - "radius_acct_multi_session_id", - "radius_service_type", - "radius_acct_authentic" - ], - "metrics": [ - "common_start_time", - "common_end_time", - "common_start_timestamp_ms", - "common_end_timestamp_ms", - "common_device_id", - "common_data_center", - "common_sled_ip", - "common_device_group", - "common_address_type", - "common_schema_type", - "common_vsys_id", - "common_client_ip", - "common_internal_ip", - "common_client_port", - "common_client_os_name", - "common_client_location", - "common_client_asn", - "common_subscriber_id", - "common_in_src_mac", - "common_out_src_mac", - "common_server_ip", - "common_external_ip", - "common_server_port", - "common_server_os_name", - "common_server_location", - "common_server_asn", - "common_server_fqdn", - "common_server_domain", - "common_in_dest_mac", - "common_out_dest_mac", - "common_l4_protocol", - "common_c2s_pkt_num", - "common_s2c_pkt_num", - "common_c2s_byte_num", - "common_s2c_byte_num", - "common_sessions", - "radius_framed_ip", - "radius_nas_ip", - "radius_account", - "radius_session_timeout", - "radius_idle_timeout", - "radius_nas_port", - "radius_event_timestamp", - "radius_acct_input_octets", - "radius_acct_output_octets", - "radius_acct_input_packets", - "radius_acct_output_packets", - "radius_acct_session_time", - "radius_acct_link_count", - "radius_acct_interim_interval" - ], - "filters": [ - "common_stream_trace_id", - "common_stream_dir", - "common_start_time", - "common_end_time", - "common_start_timestamp_ms", - "common_end_timestamp_ms", - "common_con_duration_ms", - "common_establish_latency_ms", - "common_processing_time", - "common_ingestion_time", - "common_device_id", - "common_data_center", - "common_sled_ip", - "common_device_group", - "common_address_type", - "common_vsys_id", - "common_client_ip", - "common_internal_ip", - "common_client_port", - "common_client_os_name", - "common_client_location", - "common_client_asn", - "common_subscriber_id", - "common_in_src_mac", - "common_out_src_mac", - "common_server_ip", - "common_external_ip", - "common_server_port", - "common_server_os_name", - "common_server_location", - "common_server_asn", - "common_server_fqdn", - "common_server_domain", - "common_in_dest_mac", - "common_out_dest_mac", - "common_l4_protocol", - "common_tunnel_endpoint_a_desc", - "common_tunnel_endpoint_b_desc", - "common_c2s_pkt_num", - "common_s2c_pkt_num", - "common_c2s_byte_num", - "common_s2c_byte_num", - "radius_framed_ip", - "radius_nas_ip", - "radius_account", - "radius_packet_type", - "radius_acct_status_type", - "radius_acct_terminate_cause", - "radius_called_station_id", - "radius_calling_station_id", - "radius_acct_session_id", - "radius_acct_multi_session_id", - "radius_service_type", - "radius_acct_authentic", - "radius_session_timeout", - "radius_idle_timeout", - "radius_nas_port", - "radius_event_timestamp", - "radius_acct_input_octets", - "radius_acct_output_octets", - "radius_acct_input_packets", - "radius_acct_output_packets", - "radius_acct_session_time", - "radius_acct_link_count", - "radius_acct_interim_interval" - ], - "references": { - "$ref": "public_schema_info_2311.json#/schema_query/references" - }, - "details": { - "general": [ - "common_recv_time", - "common_log_id", - "common_stream_trace_id", - "common_address_type", - "common_schema_type", - "common_vsys_id", - "common_t_vsys_id", - "common_flags", - "common_flags_identify_info", - "common_direction", - "common_stream_dir", - "common_start_time", - "common_end_time", - "common_start_timestamp_ms", - "common_end_timestamp_ms", - "common_con_duration_ms", - "common_establish_latency_ms", - "common_processing_time", - "common_ingestion_time", - "common_entrance_id", - "common_device_id", - "common_out_link_id", - "common_in_link_id", - "common_isp", - "common_data_center", - "common_device_group", - "common_sled_ip" - ], - "source": [ - "common_client_ip", - "common_internal_ip", - "common_client_port", - "common_client_os_name", - "common_client_location", - "common_client_asn", - "common_subscriber_id", - "common_imei", - "common_imsi", - "common_phone_number", - "common_in_src_mac", - "common_out_src_mac" - ], - "destination": [ - "common_server_ip", - "common_external_ip", - "common_server_port", - "common_server_os_name", - "common_server_location", - "common_server_asn", - "common_server_fqdn", - "common_server_domain", - "common_in_dest_mac", - "common_out_dest_mac" - ], - "application": [ - "common_app_id", - "common_app_full_path", - "common_userdefine_app_name", - "common_app_identify_info", - "common_app_label", - "common_app_surrogate_id", - "common_l7_protocol", - "common_protocol_label", - "common_service_category", - "common_service", - "common_l4_protocol", - "common_app_behavior" - ], - "transmission": [ - "common_sessions", - "common_c2s_pkt_num", - "common_s2c_pkt_num", - "common_c2s_byte_num", - "common_s2c_byte_num", - "common_c2s_ipfrag_num", - "common_s2c_ipfrag_num", - "common_c2s_tcp_lostlen", - "common_s2c_tcp_lostlen", - "common_c2s_tcp_unorder_num", - "common_s2c_tcp_unorder_num", - "common_c2s_pkt_retrans", - "common_s2c_pkt_retrans", - "common_c2s_byte_retrans", - "common_s2c_byte_retrans", - "common_first_ttl", - "common_tcp_client_isn", - "common_tcp_server_isn", - "common_mirrored_pkts", - "common_mirrored_bytes" - ], - "other": [ - "common_device_tag", - "common_encapsulation", - "common_tunnels", - "common_address_list", - "common_has_dup_traffic", - "common_stream_error", - "common_link_info_c2s", - "common_link_info_s2c", - "common_packet_capture_file", - "common_tunnel_endpoint_a_desc", - "common_tunnel_endpoint_b_desc", - "common_action", - "common_sub_action", - "common_policy_id", - "common_user_tags", - "common_user_region", - "common_shaping_rule_ids" - ] - } - }, - "schema_type": { - "RADIUS": { - "$ref": "public_schema_info_2311.json#/schema_type/RADIUS" - } - }, - "default_columns": [ - "common_recv_time", - "common_log_id", - "common_subscriber_id", - "radius_nas_ip", - "radius_framed_ip", - "radius_acct_status_type" - ], - "internal_columns": [ - "common_recv_time", - "common_log_id", - "common_sessions", - "common_flags_identify_info", - "common_app_identify_info", - "common_packet_capture_file", - "common_tunnel_endpoint_a_desc", - "common_tunnel_endpoint_b_desc" - ], - "tunnel_type": { - "$ref": "public_schema_info_2311.json#/tunnel_type" - } - }, - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_log_id", - "label": "Log ID", - "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_subscriber_id", - "label": "Subscriber ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_imei", - "label": "IMEI", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_imsi", - "label": "IMSI", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_phone_number", - "label": "Phone Number", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_in_src_mac", - "label": "Incoming Source MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_out_src_mac", - "label": "Outgoing Source MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_client_ip", - "label": "Client IP", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_client_port", - "label": "Client Port", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_client_os_name", - "label": "Client OS", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_internal_ip", - "label": "Internal IP", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_l4_protocol", - "label": "L4 Protocol", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_address_type", - "label": "Address Type", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_server_ip", - "label": "Server IP", - "doc": { - "constraints": { - "type": "ip" - }, - "format": [ - { - "function": "geo_asn", - "appendTo": "common_server_asn" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_port", - "label": "Server Port", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_server_os_name", - "label": "Server OS", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_external_ip", - "label": "External IP", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_action", - "label": "Action", - "doc": { - "visibility": "hidden", - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "0", - "value": "None" - }, - { - "code": "1", - "value": "Monitor" - }, - { - "code": "2", - "value": "Intercept" - }, - { - "code": "16", - "value": "Deny" - }, - { - "code": "48", - "value": "Manipulation" - }, - { - "code": "96", - "value": "Allow" - }, - { - "code": "128", - "value": "Allow(Deprecated)" - } - ], - "ttl": null - }, - "type": "int" - }, - { - "name": "common_direction", - "label": "Direction", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "69", - "value": "outbound" - }, - { - "code": "73", - "value": "inbound" - } - ], - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_sled_ip", - "label": "Sled IP", - "doc": { - "constraints": { - "type": "ip" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_client_location", - "label": "Client Location", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_client_asn", - "label": "Client ASN", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_location", - "label": "Server Location", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_asn", - "label": "Server ASN", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_fqdn", - "label": "Server FQDN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_server_domain", - "label": "Server Domain", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_in_dest_mac", - "label": "Incoming Destination MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_out_dest_mac", - "label": "Outgoing Destination MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_sessions", - "label": "Sessions", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_pkt_num", - "label": "Packets Sent", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_pkt_num", - "label": "Packets Received", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_byte_num", - "label": "Bytes Sent", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_byte_num", - "label": "Bytes Received", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_pkt_diff", - "label": "Packets Sent (Delta)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "disabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_pkt_diff", - "label": "Packets Received (Delta)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "disabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_byte_diff", - "label": "Bytes Sent (Delta)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "disabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_byte_diff", - "label": "Bytes Received (Delta)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "disabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_service", - "label": "Service", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_schema_type", - "label": "Schema Type", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "BASE", - "value": "BASE" - }, - { - "code": "HTTP", - "value": "HTTP" - }, - { - "code": "MAIL", - "value": "MAIL" - }, - { - "code": "DNS", - "value": "DNS" - }, - { - "code": "SSL", - "value": "SSL" - }, - { - "code": "FTP", - "value": "FTP" - } - ], - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_vsys_id", - "label": "Vsys ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int", - "default": 1 - }, - { - "name": "common_t_vsys_id", - "label": "Traffic Vsys ID", - "doc": { - "allow_query": "false", - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_flags", - "label": "Flags", - "doc": { - "constraints": { - "type": "bit", - "operator_functions": "=,!=,bitAnd" - }, - "data": { - "$ref": "public_schema_info_2311.json#/fields/common_flags/data" - }, - "allow_query": "false", - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "bit" - } - }, - { - "name": "common_flags_identify_info", - "label": "Flags Identify Info", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_user_tags", - "label": "User Tags", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_sub_action", - "label": "Sub Action", - "doc": { - "data": [ - { - "code": "allow", - "value": "Allow" - }, - { - "code": "deny", - "value": "Deny" - }, - { - "code": "monitor", - "value": "Monitor" - }, - { - "code": "replace", - "value": "Replace" - }, - { - "code": "redirect", - "value": "Redirect" - }, - { - "code": "insert", - "value": "Insert" - }, - { - "code": "hijack", - "value": "Hijack" - } - ], - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_user_region", - "label": "User Region", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_shaping_rule_ids", - "label": "Shaping Rule IDs", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "int", - "logicalType": "array" - } - }, - { - "name": "common_device_id", - "label": "Device ID", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_out_link_id", - "label": "Outgoing Link ID", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_in_link_id", - "label": "Incoming Link ID", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_isp", - "label": "ISP", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_device_tag", - "label": "Device Tag", - "doc": { - "visibility": "hidden", - "format": [ - { - "function": "flattenSpec", - "appendTo": "common_data_center", - "param": "$.tags[?(@.tag=='data_center')][0].value" - }, - { - "function": "flattenSpec", - "appendTo": "common_device_group", - "param": "$.tags[?(@.tag=='device_group')][0].value" - } - ], - "ttl": null - }, - "type": "string" - }, - { - "name": "common_data_center", - "label": "Data Center", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", - "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_device_group", - "label": "Device Group", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", - "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_app_behavior", - "label": "Application Behavior", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_encapsulation", - "label": "Encapsulation", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "public_schema_info_2311.json#/fields/common_encapsulation/data" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_app_label", - "label": "Application Label", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_tunnels", - "label": "Tunnels", - "doc": { - "visibility": "hidden", - "ttl": null, - "format": [ - { - "function": "gtpc_match", - "appendTo": "common_imsi,common_imei,common_phone_number", - "param": "$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_a2b_teid,$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_b2a_teid" - } - ] - }, - "type": "string" - }, - { - "name": "common_protocol_label", - "label": "Protocol Label", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_app_id", - "label": "Application ID", - "type": "string", - "doc": { - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_app_full_path", - "label": "Application Full Path", - "type": "string", - "doc": { - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_userdefine_app_name", - "label": "User Define App Name", - "type": "string", - "doc": { - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_app_identify_info", - "label": "App Identity Info", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_app_surrogate_id", - "label": "Surrogate ID", - "type": "string", - "doc": { - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_l7_protocol", - "label": "L7 Protocol", - "type": "string", - "doc": { - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_service_category", - "label": "FQDN Category", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "disabled", - "dict_location": { - "path": "/v1/category/dict", - "key": "categoryId", - "value": "categoryName" - }, - "ttl": null - }, - "type": { - "type": "array", - "items": "int", - "logicalType": "array" - } - }, - { - "name": "common_start_time", - "label": "Start Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_end_time", - "label": "End Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_start_timestamp_ms", - "label": "Start Time", - "doc": { - "constraints": { - "operator_functions": "=,!=,>,<,>=,<=", - "type": "datetime64" - }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "string", - "logicalType": "datetime64" - } - }, - { - "name": "common_end_timestamp_ms", - "label": "End Time", - "doc": { - "constraints": { - "operator_functions": "=,!=,>,<,>=,<=", - "type": "datetime64" - }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "string", - "logicalType": "datetime64" - } - }, - { - "name": "common_establish_latency_ms", - "label": "TCP Handshake Latency (ms)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_con_duration_ms", - "label": "Duration (ms)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_stream_dir", - "label": "Stream Direction(Deprecated)", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "1", - "value": "c2s" - }, - { - "code": "2", - "value": "s2c" - }, - { - "code": "3", - "value": "double" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_address_list", - "label": "Address List", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_has_dup_traffic", - "label": "Duplication Traffic", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "public_schema_info_2311.json#/fields/common_has_dup_traffic/data" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_stream_error", - "label": "Stream Error", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_stream_trace_id", - "label": "Session ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info (c2s)", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info (s2c)", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_packet_capture_file", - "label": "Packet Capture File", - "doc": { - "visibility": "hidden", - "constraints": { - "type": "file" - }, - "ttl": null - }, - "type": "string" - }, - { - "name": "common_tunnel_endpoint_a_desc", - "label": "Tunnel Endpoint A Description", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_tunnel_endpoint_b_desc", - "label": "Tunnel Endpoint B Description", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_c2s_ipfrag_num", - "label": "Fragmentation Packets (c2s)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_ipfrag_num", - "label": "Fragmentation Packets (s2c)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_tcp_lostlen", - "label": "Sequence Gap Loss (c2s)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_tcp_lostlen", - "label": "Sequence Gap Loss (s2c)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_tcp_unorder_num", - "label": "Unordered Packets (c2s)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_tcp_unorder_num", - "label": "Unordered Packets (s2c)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_pkt_retrans", - "label": "Packet Retransmission (c2s)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_pkt_retrans", - "label": "Packet Retransmission (s2c)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_c2s_byte_retrans", - "label": "Byte Retransmission (c2s)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_s2c_byte_retrans", - "label": "Byte Retransmission (s2c)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_tcp_client_isn", - "label": "TCP Client ISN", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_tcp_server_isn", - "label": "TCP Server ISN", - "doc": { - "visibility": "disabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "common_first_ttl", - "label": "First TTL", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_processing_time", - "label": "Processing Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "format": [ - { - "function": "current_timestamp" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_ingestion_time", - "label": "Ingestion Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "format": [ - { - "function": "ingestion_time" - }, - { - "function": "get_value", - "appendTo": "common_recv_time" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_mirrored_pkts", - "label": "Mirrored Packets", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "common_mirrored_bytes", - "label": "Mirrored Bytes", - "type": "long", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden", - "ttl": null - } - }, - { - "name": "radius_packet_type", - "label": "Packet Type", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "1", - "value": "Access-Request" - }, - { - "code": "2", - "value": "Access-Accept" - }, - { - "code": "3", - "value": "Access-Reject" - }, - { - "code": "4", - "value": "Accounting-Request" - }, - { - "code": "5", - "value": "Accounting-Response" - }, - { - "code": "11", - "value": "Access-Challenge" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "radius_account", - "label": "Account", - "doc": { - "format": [ - { - "function": "get_value", - "appendTo": "common_subscriber_id" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "radius_nas_ip", - "label": "Nas IP", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "radius_framed_ip", - "label": "Framed IP", - "doc": { - "constraints": { - "type": "ip" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "radius_session_timeout", - "label": "Session Timeout", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "radius_idle_timeout", - "label": "Idle Timeout", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "radius_acct_status_type", - "label": "ACC Status Type", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "1", - "value": "Start" - }, - { - "code": "2", - "value": "Stop" - }, - { - "code": "3", - "value": "Interim-Update" - }, - { - "code": "7", - "value": "Accounting-On" - }, - { - "code": "8", - "value": "Accounting-Off" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "radius_acct_terminate_cause", - "label": "Acct Terminate Cause", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "1", - "value": "User Request" - }, - { - "code": "2", - "value": "Lost Carrier" - }, - { - "code": "3", - "value": "Lost Service" - }, - { - "code": "4", - "value": "Idle Timeout" - }, - { - "code": "5", - "value": "Session Timeout" - }, - { - "code": "6", - "value": "Admin Reset" - }, - { - "code": "7", - "value": "Admin Reboot" - }, - { - "code": "8", - "value": "Port Error" - }, - { - "code": "9", - "value": "NAS Error" - }, - { - "code": "10", - "value": "NAS Request" - }, - { - "code": "11", - "value": "NAS Reboot" - }, - { - "code": "12", - "value": "Port Unneeded" - }, - { - "code": "13", - "value": "Port Preempted" - }, - { - "code": "14", - "value": "Port Suspended" - }, - { - "code": "15", - "value": "Service Unavailable" - }, - { - "code": "16", - "value": "Callback" - }, - { - "code": "17", - "value": "User Error" - }, - { - "code": "18", - "value": "Host Request" - } - ], - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "radius_event_timestamp", - "label": "Event Timestamp", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "enabled", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "radius_service_type", - "label": "Service Type", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "radius_nas_port", - "label": "Nas Port", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "radius_framed_protocol", - "label": "Framed Protocol", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "radius_callback_number", - "label": "Callback Number", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "radius_callback_id", - "label": "Callback ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "radius_termination_action", - "label": "Termination Action", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "radius_called_station_id", - "label": "Called Station ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "radius_calling_station_id", - "label": "Calling Station ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "radius_acct_delay_time", - "label": "Acct Delay Time", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "radius_acct_session_id", - "label": "Acct Session ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "radius_acct_multi_session_id", - "label": "Acct Multi Session ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "radius_acct_input_octets", - "label": "Acct Input Octets", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "radius_acct_output_octets", - "label": "Acct Output Octets", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "radius_acct_input_packets", - "label": "Acct Input Packets", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "radius_acct_output_packets", - "label": "Acct Output Packets", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "radius_acct_session_time", - "label": "Acct Session Time", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "radius_acct_link_count", - "label": "Acct Link Count", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "radius_acct_interim_interval", - "label": "Acct Interim Interval", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "radius_acct_authentic", - "label": "Acct Authentic", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/recommendation_app_cip.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/recommendation_app_cip.json deleted file mode 100644 index 8d815eaf..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/recommendation_app_cip.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "type": "record", - "name": "recommendation_app_cip", - "namespace": "tsg_galaxy", - "fields": [ - { - "name": "ROWKEY", - "label": "Row Key", - "type": "string" - }, - { - "name": "app_label", - "label": "APP Label", - "type": "string" - }, - { - "name": "last_update_time", - "label": "Last Update Time", - "type": "long" - }, - { - "name": "client_ip_list", - "label": "Client IP List", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/relation_account_framedip.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/relation_account_framedip.json deleted file mode 100644 index 7a071417..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/relation_account_framedip.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "type": "record", - "name": "relation_account_framedip", - "namespace": "tsg_galaxy", - "fields": [ - { - "name":"ROWKEY", - "label":"Row Key", - "type":"string" - }, - { - "name":"acct_status_type", - "label":"Acct Status Type", - "type":"string" - }, - { - "name":"first_found_time", - "label":"First Found Time", - "type":"long" - }, - { - "name":"last_update_time", - "label":"Last Update Time", - "type":"long" - }, - { - "name":"framed_ip", - "label":"Framed IP", - "type":"string" - }, - { - "name":"account", - "label":"Account", - "type":"string" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/report_result.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/saved_query_result.json index fdd1b7bd..8391dc6d 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-cn_nacos/data/config-data-tenant/cn/Galaxy/report_result.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/saved_query_result.json @@ -1,7 +1,7 @@ { "type": "record", - "name": "report_result", - "namespace": "tsg", + "name": "saved_query_result", + "namespace": "tsg_galaxy", "fields": [ { "name":"ROWKEY", diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/security_event.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/security_event.json index 5e811329..615581f0 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/security_event.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/security_event.json @@ -5,22 +5,31 @@ "doc": { "primary_key": "log_id", "partition_key": "recv_time", - "index_key": { - "$ref": "public_schema_info.json#/security_event/index_key" - }, + "index_key": [ + "vsys_id", + "security_action", + "proxy_action", + "decoded_as", + "data_center", + "device_group", + "recv_time" + ], "ttl": null, "default_ttl": 2592000, - "index_table": "", "functions": { "$ref": "public_schema_info.json#/functions" }, "schema_query": { - "dimensions": [ - "session_id", + "time": [ + "recv_time", "start_timestamp_ms", "end_timestamp_ms", "processing_time", "ingestion_time", + "insert_time" + ], + "dimensions": [ + "session_id", "device_id", "out_link_id", "in_link_id", @@ -29,9 +38,12 @@ "device_group", "address_type", "decoded_as", + "direction", "vsys_id", "flags", "flags_identify_info", + "c2s_ttl", + "s2c_ttl", "security_action", "security_rule_list", "monitor_rule_list", @@ -39,6 +51,10 @@ "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", "imei", @@ -52,17 +68,25 @@ "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", "in_dest_mac", "out_dest_mac", - "app_path", + "app_transition", "app", + "app_category", "app_debug_info", "app_content", - "protocol_path", + "app_extra_info", + "ip_protocol", + "decoded_path", "dup_traffic_flag", + "tunnel_id_list", "tunnel_endpoint_a_desc", "tunnel_endpoint_b_desc", "http_url", @@ -98,6 +122,7 @@ "mail_attachment_name", "mail_attachment_name_charset", "mail_eml_file", + "mail_starttls_flag", "dns_message_id", "dns_qr", "dns_opcode", @@ -183,48 +208,49 @@ "rdp_certificate_count", "rdp_certificate_permanent", "rdp_encryption_level", - "rdp_encryption_method" + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "internal_port_list", + "external_port_list", + "internal_asn_list", + "external_asn_list", + "security_rule_id" ], "metrics": [ - "start_timestamp_ms", - "end_timestamp_ms", - "device_id", - "data_center", - "sled_ip", - "device_group", - "address_type", - "decoded_as", - "vsys_id", - "security_action", - "security_mirrored_pkts", - "security_mirrored_bytes", + "log_id", + "c2s_ttl", + "s2c_ttl", "client_ip", "client_port", - "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", "imei", "imsi", "apn", "phone_number", - "packet_capture_file", "in_src_mac", "out_src_mac", "server_ip", "server_port", - "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", "in_dest_mac", "out_dest_mac", - "app_path", - "app_debug_info", "app", - "app_content", - "protocol_path", + "app_category", + "decoded_path", "sent_pkts", "received_pkts", "sent_bytes", @@ -246,37 +272,21 @@ "tcp_server_isn", "http_url", "http_host", - "http_request_line", - "http_response_line", - "http_proxy_flag", - "http_sequence", - "http_cookie", - "http_referer", "http_user_agent", "http_request_content_length", - "http_request_content_type", "http_response_content_length", - "http_response_content_type", - "http_set_cookie", - "http_version", "http_status_code", "http_response_latency_ms", "http_session_duration_ms", - "http_action_file_size", - "mail_protocol_type", "mail_account", "mail_from_cmd", "mail_to_cmd", "mail_from", - "mail_password", "mail_to", "mail_cc", "mail_bcc", "mail_subject", - "mail_subject_charset", "mail_attachment_name", - "mail_attachment_name_charset", - "mail_eml_file", "dns_message_id", "dns_qr", "dns_opcode", @@ -286,18 +296,9 @@ "dns_rcode", "dns_qtype", "dns_qclass", - "dns_qdcount", - "dns_ancount", - "dns_nscount", - "dns_arcount", "dns_qname", - "dns_qtype", - "dns_qclass", "dns_cname", - "dns_sub", - "dns_rr", "dns_response_latency_ms", - "ssl_version", "ssl_sni", "ssl_san", "ssl_cn", @@ -306,72 +307,20 @@ "ssl_ja3s_hash", "ssl_cert_issuer", "ssl_cert_subject", - "ssl_esni_flag", - "ssl_ech_flag", - "dtls_cookie", - "dtls_version", "dtls_sni", "dtls_san", "dtls_cn", "dtls_handshake_latency_ms", - "dtls_ja3_fingerprint", "dtls_ja3_hash", "dtls_cert_issuer", "dtls_cert_subject", "quic_sni", - "quic_version", "quic_user_agent", "ftp_account", "ftp_url", - "ftp_link_type", "sip_call_id", - "sip_originator_description", - "sip_responder_description", - "sip_user_agent", "sip_server", - "sip_originator_sdp_connect_ip", - "sip_originator_sdp_media_port", - "sip_originator_sdp_media_type", - "sip_originator_sdp_content", - "sip_responder_sdp_connect_ip", - "sip_responder_sdp_media_port", - "sip_responder_sdp_media_type", - "sip_responder_sdp_content", - "sip_duration_s", - "sip_bye", - "rtp_payload_type_c2s", - "rtp_payload_type_s2c", - "rtp_originator_dir", - "ssh_version", - "ssh_auth_success", - "ssh_client_version", - "ssh_server_version", - "ssh_cipher_alg", - "ssh_mac_alg", - "ssh_compression_alg", - "ssh_kex_alg", - "ssh_host_key_alg", - "ssh_host_key", - "ssh_hassh", - "stratum_cryptocurrency", - "stratum_mining_pools", - "stratum_mining_program", - "stratum_mining_subscribe", - "rdp_cookie", - "rdp_security_protocol", - "rdp_client_channels", - "rdp_keyboard_layout", - "rdp_client_version", - "rdp_client_name", - "rdp_client_product_id", - "rdp_desktop_width", - "rdp_desktop_height", - "rdp_requested_color_depth", - "rdp_certificate_type", - "rdp_certificate_count", - "rdp_certificate_permanent", - "rdp_encryption_level", - "rdp_encryption_method" + "ssh_hassh" ], "filters": [ "session_id", @@ -382,22 +331,31 @@ "processing_time", "ingestion_time", "device_id", + "out_link_id", + "in_link_id", "data_center", "sled_ip", "device_group", "address_type", "decoded_as", + "direction", "vsys_id", "flags", "flags_identify_info", + "c2s_ttl", + "s2c_ttl", "security_action", - "security_mirrored_pkts", - "security_mirrored_bytes", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", "security_rule_list", "client_ip", "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", "imei", @@ -410,19 +368,27 @@ "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", "in_dest_mac", "out_dest_mac", - "app_path", + "app_transition", "app_debug_info", "app", + "app_category", "app_content", + "app_extra_info", "dup_traffic_flag", + "tunnel_id_list", "tunnel_endpoint_a_desc", "tunnel_endpoint_b_desc", - "protocol_path", + "ip_protocol", + "decoded_path", "fqdn_category_list", "sent_pkts", "received_pkts", @@ -474,6 +440,7 @@ "mail_attachment_name", "mail_attachment_name_charset", "mail_eml_file", + "mail_starttls_flag", "dns_message_id", "dns_qr", "dns_opcode", @@ -488,8 +455,6 @@ "dns_nscount", "dns_arcount", "dns_qname", - "dns_qtype", - "dns_qclass", "dns_cname", "dns_sub", "dns_rr", @@ -568,13 +533,175 @@ "rdp_certificate_count", "rdp_certificate_permanent", "rdp_encryption_level", - "rdp_encryption_method" + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "internal_port_list", + "external_port_list", + "internal_asn_list", + "external_asn_list", + "security_rule_id" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" }, "details": { - "$ref": "public_schema_info.json#/details" + "general": { + "$ref": "public_schema_info.json#/details/general" + }, + "treatment": { + "$ref": "public_schema_info.json#/details/treatment" + }, + "source": { + "$ref": "public_schema_info.json#/details/source" + }, + "destination": { + "$ref": "public_schema_info.json#/details/destination" + }, + "application": { + "$ref": "public_schema_info.json#/details/application" + }, + "protocol": [ + "ip_protocol", + "decoded_path", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "ftp_account", + "ftp_url", + "ftp_link_type", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "mail_starttls_flag", + "quic_version", + "quic_sni", + "quic_user_agent", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe" + ], + "transmission": { + "$ref": "public_schema_info.json#/details/transmission" + }, + "other": { + "$ref": "public_schema_info.json#/details/other" + } } }, "decoded_as": { @@ -608,6 +735,9 @@ "RTP": { "$ref": "public_schema_info.json#/decoded_as/RTP" }, + "RDP": { + "$ref": "public_schema_info.json#/decoded_as/RDP" + }, "SSH": { "$ref": "public_schema_info.json#/decoded_as/SSH" }, @@ -629,7 +759,7 @@ "recv_time", "log_id", "flags_identify_info", - "tunnels", + "encapsulation", "app_debug_info", "app_content", "packet_capture_file", @@ -640,7 +770,195 @@ "$ref": "public_schema_info.json#/tunnel_type" }, "measurements": { - "aggregates": { + "aggregated_metric_unit": { + "SUM": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes" + }, + "RATE": { + "sent_pkts": "pps", + "received_pkts": "pps", + "sent_bytes": "Bps", + "received_bytes": "Bps", + "tcp_c2s_lost_bytes": "Bps", + "tcp_s2c_lost_bytes": "Bps", + "tcp_c2s_o3_pkts": "pps", + "tcp_s2c_o3_pkts": "pps", + "tcp_c2s_rtx_pkts": "pps", + "tcp_s2c_rtx_pkts": "pps", + "tcp_c2s_rtx_bytes": "Bps", + "tcp_s2c_rtx_bytes": "Bps", + "http_request_content_length": "Bps", + "http_response_content_length": "Bps" + }, + "BITRATE": { + "sent_bytes": "bps", + "received_bytes": "bps", + "tcp_c2s_lost_bytes": "bps", + "tcp_s2c_lost_bytes": "bps", + "tcp_c2s_rtx_bytes": "bps", + "tcp_s2c_rtx_bytes": "bps", + "http_request_content_length": "bps", + "http_response_content_length": "bps" + }, + "MAX": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MIN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "AVG": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MEDIAN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P95_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P99_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + } + }, + "field_discovery_metric": { "sessions": [ { "fn": "count", @@ -678,7 +996,108 @@ } ] } - } + }, + "expression_fields": [ + { + "name": "internal_ip_list", + "label": "Internal IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_ip,''), if(bitAnd(flags, 16)=16, server_ip,''))" + } + }, + { + "name": "external_ip_list", + "label": "External IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_ip,''), if(bitAnd(flags, 16)!=16, server_ip,''))" + } + }, + { + "name": "internal_port_list", + "label": "Internal Port List", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_port, null), if(bitAnd(flags, 16)=16, server_port, null))" + } + }, + { + "name": "external_port_list", + "label": "External Port List", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_port, null), if(bitAnd(flags, 16)!=16, server_port, null))" + } + }, + { + "name": "internal_asn_list", + "label": "Internal ASN List", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_asn, null), if(bitAnd(flags, 16)=16, server_asn, null))" + } + }, + { + "name": "external_asn_list", + "label": "External ASN List", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_asn, null), if(bitAnd(flags, 16)!=16, server_asn, null))" + } + }, + { + "name": "security_rule_id", + "label": "Security Rule ID", + "type": "long", + "doc": { + "allow_query": "false", + "expression": "arrayJoin(IF(empty(security_rule_list), arrayPushFront(security_rule_list, null),security_rule_list))" + } + } + ] }, "fields": [ { @@ -700,11 +1119,9 @@ "name": "log_id", "label": "Log ID", "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], + "constraints": { + "aggregation_functions": "COUNT" + }, "visibility": "enabled", "ttl": null }, @@ -715,7 +1132,7 @@ "label": "Decoded AS", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -793,11 +1210,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -814,11 +1226,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -829,10 +1236,11 @@ }, { "name": "duration_ms", - "label": "Duration (ms)", + "label": "Duration(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -841,10 +1249,11 @@ }, { "name": "tcp_handshake_latency_ms", - "label": "TCP Handshake Latency (ms)", + "label": "TCP Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -858,11 +1267,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "current_timestamp" - } - ], "visibility": "enabled", "ttl": null }, @@ -878,15 +1282,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "ingestion_time" - }, - { - "function": "get_value", - "appendTo": "recv_time" - } - ], "visibility": "enabled", "ttl": null }, @@ -942,18 +1337,6 @@ "label": "Device Tag", "doc": { "visibility": "hidden", - "format": [ - { - "function": "flattenSpec", - "appendTo": "data_center", - "param": "$.tags[?(@.tag=='data_center')][0].value" - }, - { - "function": "flattenSpec", - "appendTo": "device_group", - "param": "$.tags[?(@.tag=='device_group')][0].value" - } - ], "ttl": null }, "type": "string" @@ -963,7 +1346,7 @@ "label": "Data Center", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -980,7 +1363,7 @@ "label": "Device Group", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -997,6 +1380,7 @@ "label": "Sled IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, "visibility": "enabled", @@ -1011,22 +1395,30 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/address_type/data" + }, "visibility": "enabled", "ttl": null }, "type": "int" }, { + "name": "direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/direction/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { "name": "vsys_id", "label": "Vsys ID", "doc": { @@ -1075,6 +1467,30 @@ "type": "string" }, { + "name": "c2s_ttl", + "label": "Client-to-Server TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { + "name": "s2c_ttl", + "label": "Server-to-Client TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { "name": "security_rule_list", "label": "Security Rule List", "doc": { @@ -1096,18 +1512,11 @@ "doc": { "visibility": "enabled", "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/security_action/data" }, - "data": [ - { - "code": "Deny", - "value": "Deny" - }, - { - "code": "Allow", - "value": "Allow" - } - ], "ttl": null }, "type": "string" @@ -1145,8 +1554,24 @@ } }, { + "name": "statistics_rule_list", + "label": "Statistics Rule List", + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "hidden", + "ttl": null + }, + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + } + }, + { "name": "sc_rsp_raw", - "label": "Service Chaining Rendered Service Path (Raw)", + "label": "Service Chaining Rendered Service Path(Raw)", "doc": { "constraints": { "operator_functions": "has,notEmpty,empty" @@ -1162,7 +1587,7 @@ }, { "name": "sc_rsp_decrypted", - "label": "Service Chaining Rendered Service Path (Decrypted)", + "label": "Service Chaining Rendered Service Path(Decrypted)", "doc": { "constraints": { "operator_functions": "has,notEmpty,empty" @@ -1214,7 +1639,7 @@ "doc": { "visibility": "hidden", "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -1234,20 +1659,12 @@ "name": "proxy_pinning_status", "label": "Proxy Pinning Status", "doc": { - "data": [ - { - "code": "0", - "value": "not pinning" - }, - { - "code": "1", - "value": "pinning" - }, - { - "code": "2", - "value": "maybe pinning" - } - ], + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/proxy_pinning_status/data" + }, "visibility": "hidden", "ttl": null }, @@ -1257,20 +1674,12 @@ "name": "proxy_intercept_status", "label": "Proxy Intercept Status", "doc": { - "data": [ - { - "code": "0", - "value": "passthrough" - }, - { - "code": "1", - "value": "intercept" - }, - { - "code": "2", - "value": "shutdown" - } - ], + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/proxy_intercept_status/data" + }, "visibility": "hidden", "ttl": null }, @@ -1287,7 +1696,7 @@ }, { "name": "proxy_server_side_latency_ms", - "label": "Proxy Server Side Latency", + "label": "Proxy Server-Side Latency(ms)", "doc": { "visibility": "hidden", "ttl": null @@ -1296,7 +1705,7 @@ }, { "name": "proxy_client_side_latency_ms", - "label": "Proxy Client Side Latency", + "label": "Proxy Client-Side Latency(ms)", "doc": { "visibility": "hidden", "ttl": null @@ -1305,7 +1714,7 @@ }, { "name": "proxy_client_side_version", - "label": "Proxy Client Side Version", + "label": "Proxy Client-Side Version", "doc": { "visibility": "hidden", "ttl": null @@ -1314,7 +1723,7 @@ }, { "name": "proxy_server_side_version", - "label": "Proxy Server Side Version", + "label": "Proxy Server-Side Version", "doc": { "visibility": "hidden", "ttl": null @@ -1323,7 +1732,7 @@ }, { "name": "proxy_cert_verify", - "label": "Proxy Cert Verify", + "label": "Proxy Certificate Verify", "doc": { "visibility": "hidden", "ttl": null @@ -1340,26 +1749,26 @@ "type": "string" }, { - "name": "security_mirrored_pkts", - "label": "Security Mirrored Packets", + "name": "monitor_mirrored_pkts", + "label": "Monitor Mirrored Packets", "type": "int", "doc": { "constraints": { "type": "decimal" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null } }, { - "name": "security_mirrored_bytes", - "label": "Security Mirrored Bytes", + "name": "monitor_mirrored_bytes", + "label": "Monitor Mirrored Bytes", "type": "int", "doc": { "constraints": { "type": "bytes" }, - "visibility": "enabled", + "visibility": "hidden", "ttl": null } }, @@ -1368,18 +1777,9 @@ "label": "Client IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "client_asn" - }, - { - "function": "radius_match", - "appendTo": "subscriber_id" - } - ], "visibility": "enabled", "ttl": null }, @@ -1389,6 +1789,10 @@ "name": "client_port", "label": "Client Port", "doc": { + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1405,7 +1809,52 @@ }, { "name": "client_geolocation", - "label": "Client Geolocation", + "label": "Client Geolocation(Deprecated)", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_country", + "label": "Client Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_super_administrative_area", + "label": "Client Super Administrative Area", + "doc": { + "constraints": { + "type": "super_administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_administrative_area", + "label": "Client Administrative Area", + "doc": { + "constraints": { + "type": "administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_sub_administrative_area", + "label": "Client SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1416,6 +1865,9 @@ "name": "client_asn", "label": "Client ASN", "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1471,14 +1923,9 @@ "label": "Server IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "server_asn" - } - ], "visibility": "enabled", "ttl": null }, @@ -1488,6 +1935,10 @@ "name": "server_port", "label": "Server Port", "doc": { + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1504,7 +1955,52 @@ }, { "name": "server_geolocation", - "label": "Server Geolocation", + "label": "Server Geolocation(Deprecated)", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_country", + "label": "Server Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_super_administrative_area", + "label": "Server Super Administrative Area", + "doc": { + "constraints": { + "type": "super_administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_administrative_area", + "label": "Server Administrative Area", + "doc": { + "constraints": { + "type": "administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_sub_administrative_area", + "label": "Server SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1515,6 +2011,9 @@ "name": "server_asn", "label": "Server ASN", "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1524,6 +2023,9 @@ "name": "server_fqdn", "label": "Server FQDN", "doc": { + "constraints": { + "type": "fqdn" + }, "visibility": "enabled", "ttl": null }, @@ -1533,14 +2035,17 @@ "name": "server_domain", "label": "Server Domain", "doc": { + "constraints": { + "type": "domain" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "app_path", - "label": "Application Path", + "name": "app_transition", + "label": "Application Transition", "type": "string", "doc": { "visibility": "enabled", @@ -1551,6 +2056,18 @@ "name": "app", "label": "Application", "doc": { + "constraints": { + "type": "app" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "app_category", + "label": "Application Category", + "doc": { "visibility": "enabled", "ttl": null }, @@ -1560,7 +2077,7 @@ "name": "app_debug_info", "label": "Application Debug Info", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1575,8 +2092,32 @@ "type": "string" }, { - "name": "protocol_path", - "label": "Protocol Path", + "name": "app_extra_info", + "label": "Application Extra Info", + "doc": { + "visibility": "hidden", + "ttl": null + }, + "type": "string" + }, + { + "name": "ip_protocol", + "label": "IP Protocol", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/ip_protocol/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "decoded_path", + "label": "Decoded Path", "doc": { "visibility": "enabled", "ttl": null @@ -1591,16 +2132,16 @@ "operator_functions": "has,notEmpty,empty" }, "dict_location": { - "path": "/v1/category/dict", - "key": "categoryId", - "value": "categoryName" + "path": "/v1/policy/object?type=fqdn_category", + "key": "category_id", + "value": "category_name" }, "visibility": "enabled", "ttl": null }, "type": { "type": "array", - "items": "int", + "items": "long", "logicalType": "array" } }, @@ -1654,7 +2195,7 @@ }, { "name": "tcp_c2s_ip_fragments", - "label": "TCP C2S IP Fragments ", + "label": "Client-to-Server IP Fragments", "doc": { "constraints": { "type": "decimal" @@ -1666,7 +2207,7 @@ }, { "name": "tcp_s2c_ip_fragments", - "label": "TCP S2C IP Fragments", + "label": "Server-to-Client IP Fragments", "doc": { "constraints": { "type": "decimal" @@ -1678,7 +2219,7 @@ }, { "name": "tcp_c2s_lost_bytes", - "label": "TCP C2S Lost Bytes", + "label": "Client-to-Server Lost Bytes", "doc": { "constraints": { "type": "bytes" @@ -1690,7 +2231,7 @@ }, { "name": "tcp_s2c_lost_bytes", - "label": "TCP S2C Lost Bytes", + "label": "Server-to-Client Lost Bytes", "doc": { "constraints": { "type": "bytes" @@ -1702,7 +2243,7 @@ }, { "name": "tcp_c2s_o3_pkts", - "label": "TCP C2S out-of-order packets", + "label": "Client-to-Server Out-of-Order Packets", "doc": { "constraints": { "type": "decimal" @@ -1714,7 +2255,7 @@ }, { "name": "tcp_s2c_o3_pkts", - "label": "TCP S2C out-of-order packets", + "label": "Server-to-Client Out-of-Order Packets", "doc": { "constraints": { "type": "decimal" @@ -1726,7 +2267,7 @@ }, { "name": "tcp_c2s_rtx_pkts", - "label": "TCP C2S Retransmission Packets", + "label": "Client-to-Server Retransmission Packets", "doc": { "constraints": { "type": "decimal" @@ -1738,7 +2279,7 @@ }, { "name": "tcp_s2c_rtx_pkts", - "label": "TCP S2C Retransmission Packets", + "label": "Server-to-Client Retransmission Packets", "doc": { "constraints": { "type": "decimal" @@ -1750,7 +2291,7 @@ }, { "name": "tcp_c2s_rtx_bytes", - "label": "TCP C2S Retransmission Bytes ", + "label": "Client-to-Server Retransmission Bytes", "doc": { "constraints": { "type": "bytes" @@ -1762,7 +2303,7 @@ }, { "name": "tcp_s2c_rtx_bytes", - "label": "TCP S2C Retransmission Bytes", + "label": "Server-to-Client Retransmission Bytes", "doc": { "constraints": { "type": "bytes" @@ -1774,10 +2315,11 @@ }, { "name": "tcp_rtt_ms", - "label": "Round-trip Time", + "label": "Round-trip Time(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -1786,7 +2328,7 @@ }, { "name": "tcp_client_isn", - "label": "TCP Client ISN", + "label": "Client ISN", "doc": { "visibility": "enabled", "ttl": null @@ -1795,7 +2337,7 @@ }, { "name": "tcp_server_isn", - "label": "TCP Server ISN", + "label": "Server ISN", "doc": { "visibility": "enabled", "ttl": null @@ -1852,18 +2394,11 @@ "type": "string" }, { - "name": "tunnels", - "label": "Tunnels", + "name": "encapsulation", + "label": "Encapsulation", "doc": { "visibility": "enabled", - "ttl": null, - "format": [ - { - "function": "gtpc_match", - "appendTo": "imsi,imei,phone_number", - "param": "$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_a2b_teid,$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_b2a_teid" - } - ] + "ttl": null }, "type": "string" }, @@ -1883,8 +2418,21 @@ "type": "int" }, { + "name": "tunnel_id_list", + "label": "Tunnel ID List", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + } + }, + { "name": "tunnel_endpoint_a_desc", - "label": "Tunnel Endpoint A Description", + "label": "Tunnel Endpoint A Description(Deprecated)", "doc": { "visibility": "enabled", "ttl": null @@ -1893,7 +2441,7 @@ }, { "name": "tunnel_endpoint_b_desc", - "label": "Tunnel Endpoint B Description", + "label": "Tunnel Endpoint B Description(Deprecated)", "doc": { "visibility": "enabled", "ttl": null @@ -1904,6 +2452,9 @@ "name": "http_url", "label": "HTTP.URL", "doc": { + "constraints": { + "type": "url" + }, "visibility": "enabled", "ttl": null }, @@ -1938,16 +2489,16 @@ }, { "name": "http_request_content_length", - "label": "HTTP.Request Content Length", + "label": "HTTP.Request Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_request_content_type", - "label": "HTTP.Request Content Type", + "label": "HTTP.Request Content-Type", "doc": { "visibility": "enabled", "ttl": null @@ -1956,12 +2507,12 @@ }, { "name": "http_response_content_length", - "label": "HTTP.Response Content Length", + "label": "HTTP.Response Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_response_content_type", @@ -1976,11 +2527,6 @@ "name": "http_request_body", "label": "HTTP.Request Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -1994,11 +2540,6 @@ "name": "http_response_body", "label": "HTTP.Response Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -2012,7 +2553,7 @@ "name": "http_proxy_flag", "label": "HTTP.Proxy Flag", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "int" @@ -2046,7 +2587,7 @@ }, { "name": "http_user_agent", - "label": "HTTP.User Agent", + "label": "HTTP.User-Agent", "doc": { "visibility": "enabled", "ttl": null @@ -2055,7 +2596,7 @@ }, { "name": "http_set_cookie", - "label": "HTTP.Set Cookie", + "label": "HTTP.Set-Cookie", "doc": { "visibility": "enabled", "ttl": null @@ -2076,21 +2617,26 @@ "label": "HTTP.Status Code", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT", + "type": "http_code" + }, "ttl": null }, "type": "int" }, { "name": "http_response_latency_ms", - "label": "HTTP.Response Latency (ms)", + "label": "HTTP.Response Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "http_action_file_size", @@ -2102,23 +2648,24 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "http_session_duration_ms", - "label": "HTTP.Session Duration (ms)", + "label": "HTTP.Session Duration(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "mail_protocol_type", - "label": "Mail.Protocol Type", + "label": "MAIL.Protocol Type", "doc": { "visibility": "enabled", "ttl": null @@ -2127,8 +2674,11 @@ }, { "name": "mail_account", - "label": "Mail.Account", + "label": "MAIL.Account", "doc": { + "constraints": { + "type": "email" + }, "visibility": "enabled", "ttl": null }, @@ -2136,7 +2686,7 @@ }, { "name": "mail_from_cmd", - "label": "Mail.From CMD", + "label": "MAIL.From CMD", "doc": { "visibility": "enabled", "ttl": null @@ -2145,7 +2695,7 @@ }, { "name": "mail_to_cmd", - "label": "Mail.To CMD", + "label": "MAIL.To CMD", "doc": { "visibility": "enabled", "ttl": null @@ -2154,7 +2704,7 @@ }, { "name": "mail_from", - "label": "Mail.From", + "label": "MAIL.From", "doc": { "constraints": { "type": "email" @@ -2166,7 +2716,7 @@ }, { "name": "mail_password", - "label": "Mail.Password", + "label": "MAIL.Password", "doc": { "visibility": "enabled", "ttl": null @@ -2175,7 +2725,7 @@ }, { "name": "mail_to", - "label": "Mail.To", + "label": "MAIL.To", "doc": { "constraints": { "type": "email" @@ -2187,7 +2737,7 @@ }, { "name": "mail_cc", - "label": "Mail.CC", + "label": "MAIL.CC", "doc": { "visibility": "enabled", "ttl": null @@ -2196,7 +2746,7 @@ }, { "name": "mail_bcc", - "label": "Mail.BCC", + "label": "MAIL.BCC", "doc": { "visibility": "enabled", "ttl": null @@ -2205,14 +2755,8 @@ }, { "name": "mail_subject", - "label": "Mail.Subject", + "label": "MAIL.Subject", "doc": { - "format": [ - { - "function": "decode_of_base64", - "param": "$.mail_subject_charset" - } - ], "visibility": "enabled", "ttl": null }, @@ -2220,7 +2764,7 @@ }, { "name": "mail_subject_charset", - "label": "Mail.Subject Charset", + "label": "MAIL.Subject Charset", "doc": { "visibility": "enabled", "ttl": null @@ -2229,14 +2773,8 @@ }, { "name": "mail_attachment_name", - "label": "Mail.Attachment", + "label": "MAIL.Attachment Name", "doc": { - "format": [ - { - "function": "decode_of_base64", - "param": "$.mail_attachment_name_charset" - } - ], "visibility": "enabled", "ttl": null }, @@ -2244,7 +2782,7 @@ }, { "name": "mail_attachment_name_charset", - "label": "Mail.Attachment Charset", + "label": "MAIL.Attachment Name Charset", "doc": { "visibility": "enabled", "ttl": null @@ -2253,13 +2791,8 @@ }, { "name": "mail_eml_file", - "label": "Mail.EML File", + "label": "MAIL.EML File", "doc": { - "format": [ - { - "function": "combine" - } - ], "constraints": { "type": "file" }, @@ -2269,6 +2802,21 @@ "type": "string" }, { + "name": "mail_starttls_flag", + "label": "MAIL.STARTTLS Flag", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/mail_starttls_flag/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { "name": "dns_message_id", "label": "DNS.Message ID", "doc": { @@ -2284,16 +2832,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "QUERY" - }, - { - "code": "1", - "value": "RESPONSE" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_qr/data" + }, "visibility": "enabled", "ttl": null }, @@ -2304,26 +2845,12 @@ "label": "DNS.OPCODE", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": { + "$ref": "public_schema_info.json#/fields/dns_opcode/data" }, - "data": [ - { - "code": "0", - "value": "QUERY" - }, - { - "code": "1", - "value": "IQUERY" - }, - { - "code": "2", - "value": "STATUS" - }, - { - "code": "5", - "value": "UPDATE" - } - ], "visibility": "enabled", "ttl": null }, @@ -2334,6 +2861,9 @@ "label": "DNS.AA", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -2369,77 +2899,14 @@ "name": "dns_rcode", "label": "DNS.RCODE", "doc": { - "data": [ - { - "code": 0, - "value": "NoError" - }, - { - "code": 1, - "value": "FormErr" - }, - { - "code": 2, - "value": "ServFail" - }, - { - "code": 3, - "value": "NXDomain" - }, - { - "code": 4, - "value": "NotImp" - }, - { - "code": 5, - "value": "Refused" - }, - { - "code": 6, - "value": "YXDomain" - }, - { - "code": 7, - "value": "YXRRSet" - }, - { - "code": 8, - "value": "NXRRSet" - }, - { - "code": 9, - "value": "NotAuth" - }, - { - "code": 10, - "value": "NotZone" - }, - { - "code": 16, - "value": "BADSIG" - }, - { - "code": 17, - "value": "BADKEY" - }, - { - "code": 18, - "value": "BADTIME" - }, - { - "code": 19, - "value": "BADMODE" - }, - { - "code": 20, - "value": "BADNAME" - }, - { - "code": 21, - "value": "BADALG" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_rcode/data" + }, "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -2506,330 +2973,12 @@ "label": "DNS.QTYPE", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": { + "$ref": "public_schema_info.json#/fields/dns_qtype/data" }, - "data": [ - { - "code": "1", - "value": "A" - }, - { - "code": "2", - "value": "NS" - }, - { - "code": "3", - "value": "MD" - }, - { - "code": "4", - "value": "MF" - }, - { - "code": "5", - "value": "CNAME" - }, - { - "code": "6", - "value": "SOA" - }, - { - "code": "7", - "value": "MB" - }, - { - "code": "8", - "value": "MG" - }, - { - "code": "9", - "value": "MR" - }, - { - "code": "10", - "value": "NULL" - }, - { - "code": "11", - "value": "WKS" - }, - { - "code": "12", - "value": "PTR" - }, - { - "code": "13", - "value": "HINFO" - }, - { - "code": "14", - "value": "MINFO" - }, - { - "code": "15", - "value": "MX" - }, - { - "code": "16", - "value": "TXT" - }, - { - "code": "17", - "value": "RP" - }, - { - "code": "18", - "value": "AFSDB" - }, - { - "code": "19", - "value": "X25" - }, - { - "code": "20", - "value": "ISDN" - }, - { - "code": "21", - "value": "RT" - }, - { - "code": "22", - "value": "NSAP" - }, - { - "code": "23", - "value": "NSAP" - }, - { - "code": "24", - "value": "SIG" - }, - { - "code": "25", - "value": "KEY" - }, - { - "code": "26", - "value": "PX" - }, - { - "code": "27", - "value": "GPOS" - }, - { - "code": "28", - "value": "AAAA" - }, - { - "code": "29", - "value": "LOC" - }, - { - "code": "30", - "value": "EID" - }, - { - "code": "31", - "value": "NIMLOC" - }, - { - "code": "32", - "value": "NB" - }, - { - "code": "33", - "value": "SRV" - }, - { - "code": "34", - "value": "ATMA" - }, - { - "code": "35", - "value": "NAPTR" - }, - { - "code": "36", - "value": "KX" - }, - { - "code": "37", - "value": "CERT" - }, - { - "code": "38", - "value": "A6" - }, - { - "code": "39", - "value": "DNAME" - }, - { - "code": "40", - "value": "SINK" - }, - { - "code": "41", - "value": "OPT" - }, - { - "code": "42", - "value": "APL" - }, - { - "code": "43", - "value": "DS" - }, - { - "code": "44", - "value": "SSHFP" - }, - { - "code": "45", - "value": "IPSECKEY" - }, - { - "code": "46", - "value": "RRSIG" - }, - { - "code": "47", - "value": "NSEC" - }, - { - "code": "48", - "value": "DNSKEY" - }, - { - "code": "49", - "value": "DHCID" - }, - { - "code": "50", - "value": "NSEC3" - }, - { - "code": "51", - "value": "NSEC3PARAM" - }, - { - "code": "52", - "value": "TLSA" - }, - { - "code": "53", - "value": "SMIMEA" - }, - { - "code": "55", - "value": "HIP" - }, - { - "code": "59", - "value": "CDS" - }, - { - "code": "60", - "value": "CDNSKEY" - }, - { - "code": "61", - "value": "OPENPGPKEY" - }, - { - "code": "62", - "value": "CSYNC" - }, - { - "code": "63", - "value": "ZONEMD" - }, - { - "code": "64", - "value": "SVCB" - }, - { - "code": "65", - "value": "HTTPS" - }, - { - "code": "99", - "value": "SPF" - }, - { - "code": "100", - "value": "UINFO" - }, - { - "code": "101", - "value": "UID" - }, - { - "code": "102", - "value": "GID" - }, - { - "code": "103", - "value": "UNSPEC" - }, - { - "code": "108", - "value": "EUI48" - }, - { - "code": "109", - "value": "EUI64" - }, - { - "code": "249", - "value": "TKEY" - }, - { - "code": "250", - "value": "TSIG" - }, - { - "code": "251", - "value": "IXFR" - }, - { - "code": "252", - "value": "AXFR" - }, - { - "code": "253", - "value": "MAILB" - }, - { - "code": "254", - "value": "MAILA" - }, - { - "code": "255", - "value": "*" - }, - { - "code": "256", - "value": "URI" - }, - { - "code": "257", - "value": "CAA" - }, - { - "code": "32768", - "value": "TA" - }, - { - "code": "32769", - "value": "DLV" - }, - { - "code": "65521", - "value": "INTEGRITY" - } - ], "visibility": "enabled", "ttl": null }, @@ -2840,6 +2989,10 @@ "label": "DNS.QCLASS", "doc": { "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -2860,16 +3013,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "1", - "value": "DNS" - }, - { - "code": "2", - "value": "DNSSEC" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_sub/data" + }, "visibility": "enabled", "ttl": null }, @@ -2886,12 +3032,13 @@ }, { "name": "dns_response_latency_ms", - "label": "DNS.Response Latency (ms)", + "label": "DNS.Response Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "int" @@ -2934,10 +3081,11 @@ }, { "name": "ssl_handshake_latency_ms", - "label": "SSL.Handshake Latency (ms)", + "label": "SSL.Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -2946,7 +3094,7 @@ }, { "name": "ssl_ja3_hash", - "label": "SSL.JA3 hash", + "label": "SSL.JA3 Hash", "doc": { "visibility": "enabled", "ttl": null @@ -2955,7 +3103,7 @@ }, { "name": "ssl_ja3s_hash", - "label": "SSL.JA3S hash", + "label": "SSL.JA3S Hash", "doc": { "visibility": "enabled", "ttl": null @@ -2990,6 +3138,9 @@ "name": "ssl_esni_flag", "label": "SSL.ESNI Flag", "doc": { + "data": { + "$ref": "public_schema_info.json#/fields/ssl_esni_flag/data" + }, "visibility": "enabled", "ttl": null }, @@ -2999,6 +3150,9 @@ "name": "ssl_ech_flag", "label": " SSL.ECH Flag", "doc": { + "data": { + "$ref": "public_schema_info.json#/fields/ssl_ech_flag/data" + }, "visibility": "enabled", "ttl": null }, @@ -3008,7 +3162,7 @@ "name": "dtls_cookie", "label": "DTLS.Cookie", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" @@ -3017,7 +3171,7 @@ "name": "dtls_version", "label": "DTLS.Version", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" @@ -3035,7 +3189,7 @@ "name": "dtls_san", "label": "DTLS.SAN", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" @@ -3044,62 +3198,63 @@ "name": "dtls_cn", "label": "DTLS.CN", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_handshake_latency_ms", - "label": "DTLS.Handshake Latency (ms)", + "label": "DTLS.Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "int" }, { "name": "dtls_ja3_fingerprint", - "label": "DTLS.JA3", + "label": "DTLS.JA3 Fingerprint", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_ja3_hash", - "label": "DTLS.JA3 hash", + "label": "DTLS.JA3 Hash", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_cert_issuer", - "label": "DTLS.Issuer", + "label": "DTLS.Certificate Issuer", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_cert_subject", - "label": "DTLS.Subject", + "label": "DTLS.Certificate Subject", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "quic_version", - "label": "Quic.Version", + "label": "QUIC.Version", "doc": { "visibility": "enabled", "ttl": null @@ -3108,7 +3263,7 @@ }, { "name": "quic_sni", - "label": "Quic.SNI", + "label": "QUIC.SNI", "doc": { "visibility": "enabled", "ttl": null @@ -3117,7 +3272,7 @@ }, { "name": "quic_user_agent", - "label": "Quic.User Agent", + "label": "QUIC.User-Agent", "doc": { "visibility": "enabled", "ttl": null @@ -3200,6 +3355,9 @@ "name": "sip_originator_sdp_connect_ip", "label": "SIP.Originator IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -3209,6 +3367,9 @@ "name": "sip_originator_sdp_media_port", "label": "SIP.Originator Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -3236,6 +3397,9 @@ "name": "sip_responder_sdp_connect_ip", "label": "SIP.Responder IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -3245,6 +3409,9 @@ "name": "sip_responder_sdp_media_port", "label": "SIP.Responder Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -3270,7 +3437,7 @@ }, { "name": "sip_duration_s", - "label": "SIP.Duration (s)", + "label": "SIP.Duration(s)", "doc": { "constraints": { "type": "decimal" @@ -3291,121 +3458,14 @@ }, { "name": "rtp_payload_type_c2s", - "label": "RTP.Payload Type (c2s)", + "label": "RTP.Payload Type(Client-to-Server)", "doc": { "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "PCMU" - }, - { - "code": "1", - "value": "1016" - }, - { - "code": "2", - "value": "G721" - }, - { - "code": "3", - "value": "GSM" - }, - { - "code": "4", - "value": "G723" - }, - { - "code": "5", - "value": "DVI4_8000" - }, - { - "code": "6", - "value": "DVI4_16000" - }, - { - "code": "7", - "value": "LPC" - }, - { - "code": "8", - "value": "PCMA" - }, - { - "code": "9", - "value": "G722" - }, - { - "code": "10", - "value": "L16_STEREO" - }, - { - "code": "11", - "value": "L16_MONO" - }, - { - "code": "12", - "value": "QCELP" - }, - { - "code": "13", - "value": "CN" - }, - { - "code": "14", - "value": "MPA" - }, - { - "code": "15", - "value": "G728" - }, - { - "code": "16", - "value": "DVI4_11025" - }, - { - "code": "17", - "value": "DVI4_22050" - }, - { - "code": "18", - "value": "G729" - }, - { - "code": "19", - "value": "CN_OLD" - }, - { - "code": "25", - "value": "CELB" - }, - { - "code": "26", - "value": "JPEG" - }, - { - "code": "28", - "value": "NV" - }, - { - "code": "31", - "value": "H261" - }, - { - "code": "32", - "value": "MPV" - }, - { - "code": "33", - "value": "MP2T" - }, - { - "code": "34", - "value": "H263" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_payload_type_c2s/data" + }, "visibility": "enabled", "ttl": null }, @@ -3413,121 +3473,14 @@ }, { "name": "rtp_payload_type_s2c", - "label": "RTP.Payload Type (s2c)", + "label": "RTP.Payload Type(Server-to-Client)", "doc": { "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "PCMU" - }, - { - "code": "1", - "value": "1016" - }, - { - "code": "2", - "value": "G721" - }, - { - "code": "3", - "value": "GSM" - }, - { - "code": "4", - "value": "G723" - }, - { - "code": "5", - "value": "DVI4_8000" - }, - { - "code": "6", - "value": "DVI4_16000" - }, - { - "code": "7", - "value": "LPC" - }, - { - "code": "8", - "value": "PCMA" - }, - { - "code": "9", - "value": "G722" - }, - { - "code": "10", - "value": "L16_STEREO" - }, - { - "code": "11", - "value": "L16_MONO" - }, - { - "code": "12", - "value": "QCELP" - }, - { - "code": "13", - "value": "CN" - }, - { - "code": "14", - "value": "MPA" - }, - { - "code": "15", - "value": "G728" - }, - { - "code": "16", - "value": "DVI4_11025" - }, - { - "code": "17", - "value": "DVI4_22050" - }, - { - "code": "18", - "value": "G729" - }, - { - "code": "19", - "value": "CN_OLD" - }, - { - "code": "25", - "value": "CELB" - }, - { - "code": "26", - "value": "JPEG" - }, - { - "code": "28", - "value": "NV" - }, - { - "code": "31", - "value": "H261" - }, - { - "code": "32", - "value": "MPV" - }, - { - "code": "33", - "value": "MP2T" - }, - { - "code": "34", - "value": "H263" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_payload_type_s2c/data" + }, "visibility": "enabled", "ttl": null }, @@ -3553,20 +3506,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "unknown" - }, - { - "code": "1", - "value": "c2s" - }, - { - "code": "2", - "value": "s2c" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_originator_dir/data" + }, "visibility": "enabled", "ttl": null }, @@ -3637,7 +3579,7 @@ }, { "name": "ssh_kex_alg", - "label": "SSH. Key Exchange Algorithm", + "label": "SSH.Key Exchange Algorithm", "doc": { "visibility": "enabled", "ttl": null @@ -3772,7 +3714,7 @@ }, { "name": "rdp_desktop_width", - "label": "RDP. Desktop Width", + "label": "RDP.Desktop Width", "doc": { "visibility": "enabled", "ttl": null diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/security_rule_hits.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/security_rule_hits.json index fa150e02..4624f5b7 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/security_rule_hits.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/security_rule_hits.json @@ -1,211 +1,223 @@ - { - "type": "record", - "name": "security_rule_hits", - "namespace": "druid", - "doc": { - "partition_key": "__time", - "functions": { - "$ref": "public_schema_info.json#/functions" - }, - "schema_query": { - "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "rule_id", - "pinning_status", - "action" - ], - "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "rule_id", - "pinning_status", - "action", - "hit_count", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" - ], - "filters": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "rule_id", - "pinning_status", - "action", - "hit_count", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" - ], - "references": { - "$ref": "public_schema_info.json#/schema_query/references" - } - }, - "measurements": { - "granularity": 1, - "ingestion_delay": 15 + "type": "record", + "name": "security_rule_hits", + "namespace": "druid", + "doc": { + "partition_key": "__time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "time": [ + "__time" + ], + "dimensions": [ + "rule_id", + "action", + "device_group", + "data_center" + ], + "metrics": [ + "hit_count", + "in_bytes", + "out_bytes", + "in_pkts", + "out_pkts" + ], + "filters": [ + "rule_id", + "action", + "device_group", + "data_center" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" } }, - "fields": [ - { - "name": "__time", - "label": "Time", - "type": { - "type": "string", - "logicalType": "timestamp" + "measurements": { + "ingestion_period": "PT1S", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "hit_count": "sessions", + "in_bytes": "bytes", + "out_bytes": "bytes", + "in_pkts": "packets", + "out_pkts": "packets" }, - "doc": { - "constraints": { - "type": "timestamp" - }, - "visibility": "enabled" + "RATE": { + "hit_count": "sps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps" } + } + } + }, + "fields": [ + { + "name": "__time", + "label": "Time", + "type": { + "type": "string", + "logicalType": "timestamp" }, - { - "name": "vsys_id", - "label": "Vsys ID", - "doc": { - "visibility": "enabled" + "doc": { + "constraints": { + "type": "timestamp" }, - "type": "int", - "default": 1 + "visibility": "enabled" + } + }, + { + "name": "vsys_id", + "label": "Vsys ID", + "doc": { + "visibility": "enabled" }, - { - "name": "device_id", - "label": "Device ID", - "doc": { - "visibility": "enabled" - }, - "type": "string" + "type": "long", + "default": 1 + }, + { + "name": "device_id", + "label": "Device ID", + "doc": { + "visibility": "enabled" }, - { - "name": "device_group", - "label": "Device Group", - "doc": { - "constraints": { - "operator_functions": "=,in" - }, - "data": { - "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", - "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" - }, - "visibility": "enabled" + "type": "string" + }, + { + "name": "device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" }, - "type": "string" + "visibility": "enabled" }, - { - "name": "data_center", - "label": "Data Center", - "doc": { - "constraints": { - "operator_functions": "=,in" - }, - "data": { - "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", - "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" - }, - "visibility": "enabled" + "type": "string" + }, + { + "name": "data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,in" }, - "type": "string" + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "visibility": "enabled" }, - { - "name": "rule_id", - "label": "Rule", - "type": "long", - "doc": { - "constraints": { - "operator_functions": "=,in" + "type": "string" + }, + { + "name": "rule_id", + "label": "Rule", + "type": "long", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "visibility": "enabled" + } + }, + { + "name": "action", + "label": "Action", + "type": "long", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": [ + { + "code": 16, + "value": "Deny" }, - "visibility": "enabled" - } - }, - { - "name": "pinning_status", - "label": "Pinning Status", - "type": "long", - "doc": { - "visibility": "enabled" - } - }, - { - "name": "action", - "label": "Action", - "type": "long", - "doc": { - "constraints": { - "operator_functions": "=,in" + { + "code": 96, + "value": "Allow" }, - "data": [ - { - "code": "1", - "value": "Monitor" - }, - { - "code": "16", - "value": "Deny" - }, - { - "code": "96", - "value": "Allow" - }, - { - "code": "128", - "value": "Shunt" - } - ], - "visibility": "enabled" - } - }, - { - "name": "hit_count", - "label": "Hit Count", - "doc": { - "visibility": "enabled" + { + "code": 128, + "value": "Shunt" + } + ], + "visibility": "enabled" + } + }, + { + "name": "hit_count", + "label": "Hit Count", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, - "type": "long" + "visibility": "enabled" }, - { - "name": "in_bytes", - "label": "Incoming Bytes", - "doc": { - "visibility": "enabled" + "type": "long" + }, + { + "name": "in_bytes", + "label": "Incoming Bytes", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" }, - "type": "long" + "visibility": "enabled" }, - { - "name": "out_bytes", - "label": "Outgoing Bytes", - "doc": { - "visibility": "enabled" + "type": "long" + }, + { + "name": "out_bytes", + "label": "Outgoing Bytes", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" }, - "type": "long" + "visibility": "enabled" }, - { - "name": "in_pkts", - "label": "Incoming Packets", - "doc": { - "visibility": "enabled" + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, - "type": "long" + "visibility": "enabled" }, - { - "name": "out_pkts", - "label": "Outgoing Packets", - "doc": { - "visibility": "enabled" + "type": "long" + }, + { + "name": "out_pkts", + "label": "Outgoing Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, - "type": "long" - } - ] - }
\ No newline at end of file + "visibility": "enabled" + }, + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/service_chaining_rule_hits.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/service_chaining_rule_hits.json index 485eda8c..a1dc9a97 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/service_chaining_rule_hits.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/service_chaining_rule_hits.json @@ -8,48 +8,54 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "rule_id", "sff_profile_id", - "sf_profile_id" + "sf_profile_id", + "device_group", + "data_center" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "rule_id", - "sff_profile_id", - "sf_profile_id", - "sent_pkts", "sent_bytes", - "recv_pkts", - "recv_bytes" + "recv_bytes", + "sent_pkts", + "recv_pkts" ], "filters": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "rule_id", "sff_profile_id", "sf_profile_id", - "sent_pkts", - "sent_bytes", - "recv_pkts", - "recv_bytes" + "device_group", + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 1, - "ingestion_delay": 5 + "ingestion_period": "PT1S", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sent_bytes": "bytes", + "recv_bytes": "bytes", + "sent_pkts": "packets", + "recv_pkts": "packets" + }, + "RATE": { + "sent_bytes": "Bps", + "recv_bytes": "Bps", + "sent_pkts": "pps", + "recv_pkts": "pps" + }, + "BITRATE": { + "sent_bytes": "bps", + "recv_bytes": "bps" + } + } } }, "fields": [ @@ -113,7 +119,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -125,7 +131,7 @@ }, "visibility": "enabled" }, - "type": "int" + "type": "long" }, { "name": "sff_profile_id", @@ -133,7 +139,7 @@ "doc": { "visibility": "enabled" }, - "type": "int" + "type": "long" }, { "name": "sf_profile_id", @@ -141,12 +147,16 @@ "doc": { "visibility": "enabled" }, - "type": "int" + "type": "long" }, { "name": "recv_bytes", "label": "Bytes Received", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -155,6 +165,10 @@ "name": "sent_bytes", "label": "Bytes Sent", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -163,6 +177,10 @@ "name": "recv_pkts", "label": "Packets Received", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -171,6 +189,10 @@ "name": "sent_pkts", "label": "Packets Sent", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/service_function_status.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/service_function_status.json index 13114519..c7ab0a52 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/service_function_status.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/service_function_status.json @@ -8,37 +8,29 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "sf_profile_id" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "sf_profile_id", - "sf_status", "sf_latency_us" ], "filters": [ - "vsys_id", "device_id", "device_group", "data_center", - "sf_profile_id", "sf_status", - "sf_latency_us" + "sf_profile_id" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 1 + "ingestion_period": "PT1S", + "ingestion_delay": 5 } }, "fields": [ @@ -102,16 +94,19 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { "name": "sf_latency_us", - "label": "Service Function Latency (us)", + "label": "Service Function Latency(us)", "doc": { + "constraints": { + "metric_type": "gauge" + }, "visibility": "enabled" }, - "type": "int" + "type": "long" }, { "name": "sf_profile_id", @@ -119,15 +114,18 @@ "doc": { "visibility": "enabled" }, - "type": "int" + "type": "long" }, { "name": "sf_status", "label": "Service Function Status", "doc": { + "constraints": { + "metric_type": "gauge" + }, "visibility": "enabled" }, - "type": "int" + "type": "long" } ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/session_record.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/session_record.json index 1ebe00e5..eb3b30ff 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/session_record.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/session_record.json @@ -5,21 +5,31 @@ "doc": { "primary_key": "log_id", "partition_key": "recv_time", - "index_key": { - "$ref": "public_schema_info.json#/session_record/index_key" - }, + "index_key": [ + "vsys_id", + "security_action", + "proxy_action", + "decoded_as", + "data_center", + "device_group", + "recv_time" + ], "ttl": null, "default_ttl": 2592000, "functions": { "$ref": "public_schema_info.json#/functions" }, "schema_query": { - "dimensions": [ - "session_id", + "time": [ + "recv_time", "start_timestamp_ms", "end_timestamp_ms", "processing_time", "ingestion_time", + "insert_time" + ], + "dimensions": [ + "session_id", "device_id", "out_link_id", "in_link_id", @@ -28,14 +38,18 @@ "device_group", "address_type", "decoded_as", + "direction", "vsys_id", "flags", "flags_identify_info", + "c2s_ttl", + "s2c_ttl", "security_action", "security_rule_list", "monitor_rule_list", "shaping_rule_list", "sc_rule_list", + "statistics_rule_list", "sc_rsp_raw", "sc_rsp_decrypted", "proxy_action", @@ -44,6 +58,10 @@ "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", "imei", @@ -57,17 +75,25 @@ "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", "in_dest_mac", "out_dest_mac", - "app_path", + "app_transition", "app", + "app_category", "app_debug_info", "app_content", - "protocol_path", + "app_extra_info", + "ip_protocol", + "decoded_path", "dup_traffic_flag", + "tunnel_id_list", "tunnel_endpoint_a_desc", "tunnel_endpoint_b_desc", "http_url", @@ -103,6 +129,7 @@ "mail_attachment_name", "mail_attachment_name_charset", "mail_eml_file", + "mail_starttls_flag", "dns_message_id", "dns_qr", "dns_opcode", @@ -188,49 +215,54 @@ "rdp_certificate_count", "rdp_certificate_permanent", "rdp_encryption_level", - "rdp_encryption_method" + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "internal_port_list", + "external_port_list", + "internal_asn_list", + "external_asn_list", + "security_rule_id", + "monitor_rule_id", + "proxy_rule_id", + "statistics_rule_id", + "shaping_rule_id", + "sc_rule_id" ], "metrics": [ - "start_timestamp_ms", - "end_timestamp_ms", - "device_id", - "data_center", - "sled_ip", - "device_group", - "address_type", - "decoded_as", - "vsys_id", - "security_action", - "proxy_action", - "security_mirrored_pkts", - "security_mirrored_bytes", + "log_id", + "c2s_ttl", + "s2c_ttl", "client_ip", "client_port", - "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", "imei", "imsi", "apn", "phone_number", - "packet_capture_file", "in_src_mac", "out_src_mac", "server_ip", "server_port", - "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", "in_dest_mac", "out_dest_mac", - "app_path", - "app_debug_info", "app", - "app_content", - "protocol_path", + "app_category", + "decoded_path", "sent_pkts", "received_pkts", "sent_bytes", @@ -250,37 +282,21 @@ "tcp_rtt_ms", "http_url", "http_host", - "http_request_line", - "http_response_line", - "http_proxy_flag", - "http_sequence", - "http_cookie", - "http_referer", "http_user_agent", "http_request_content_length", - "http_request_content_type", "http_response_content_length", - "http_response_content_type", - "http_set_cookie", - "http_version", "http_status_code", "http_response_latency_ms", "http_session_duration_ms", - "http_action_file_size", - "mail_protocol_type", "mail_account", "mail_from_cmd", "mail_to_cmd", "mail_from", - "mail_password", "mail_to", "mail_cc", "mail_bcc", "mail_subject", - "mail_subject_charset", "mail_attachment_name", - "mail_attachment_name_charset", - "mail_eml_file", "dns_message_id", "dns_qr", "dns_opcode", @@ -290,18 +306,9 @@ "dns_rcode", "dns_qtype", "dns_qclass", - "dns_qdcount", - "dns_ancount", - "dns_nscount", - "dns_arcount", "dns_qname", - "dns_qtype", - "dns_qclass", "dns_cname", - "dns_sub", - "dns_rr", "dns_response_latency_ms", - "ssl_version", "ssl_sni", "ssl_san", "ssl_cn", @@ -310,72 +317,20 @@ "ssl_ja3s_hash", "ssl_cert_issuer", "ssl_cert_subject", - "ssl_esni_flag", - "ssl_ech_flag", - "dtls_cookie", - "dtls_version", "dtls_sni", "dtls_san", "dtls_cn", "dtls_handshake_latency_ms", - "dtls_ja3_fingerprint", "dtls_ja3_hash", "dtls_cert_issuer", "dtls_cert_subject", "quic_sni", - "quic_version", "quic_user_agent", "ftp_account", "ftp_url", - "ftp_link_type", "sip_call_id", - "sip_originator_description", - "sip_responder_description", - "sip_user_agent", "sip_server", - "sip_originator_sdp_connect_ip", - "sip_originator_sdp_media_port", - "sip_originator_sdp_media_type", - "sip_originator_sdp_content", - "sip_responder_sdp_connect_ip", - "sip_responder_sdp_media_port", - "sip_responder_sdp_media_type", - "sip_responder_sdp_content", - "sip_duration_s", - "sip_bye", - "rtp_payload_type_c2s", - "rtp_payload_type_s2c", - "rtp_originator_dir", - "ssh_version", - "ssh_auth_success", - "ssh_client_version", - "ssh_server_version", - "ssh_cipher_alg", - "ssh_mac_alg", - "ssh_compression_alg", - "ssh_kex_alg", - "ssh_host_key_alg", - "ssh_host_key", - "ssh_hassh", - "stratum_cryptocurrency", - "stratum_mining_pools", - "stratum_mining_program", - "stratum_mining_subscribe", - "rdp_cookie", - "rdp_security_protocol", - "rdp_client_channels", - "rdp_keyboard_layout", - "rdp_client_version", - "rdp_client_name", - "rdp_client_product_id", - "rdp_desktop_width", - "rdp_desktop_height", - "rdp_requested_color_depth", - "rdp_certificate_type", - "rdp_certificate_count", - "rdp_certificate_permanent", - "rdp_encryption_level", - "rdp_encryption_method" + "ssh_hassh" ], "filters": [ "session_id", @@ -386,29 +341,39 @@ "processing_time", "ingestion_time", "device_id", + "out_link_id", + "in_link_id", "data_center", "sled_ip", "device_group", "address_type", "decoded_as", + "direction", "vsys_id", "flags", "flags_identify_info", + "c2s_ttl", + "s2c_ttl", "security_action", "security_rule_list", "monitor_rule_list", "shaping_rule_list", "sc_rule_list", + "statistics_rule_list", "sc_rsp_raw", "sc_rsp_decrypted", "proxy_rule_list", "proxy_action", - "security_mirrored_pkts", - "security_mirrored_bytes", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", "client_ip", "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", "subscriber_id", "imei", @@ -421,19 +386,27 @@ "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", "server_fqdn", "server_domain", "in_dest_mac", "out_dest_mac", - "app_path", + "app_transition", "app_debug_info", "app", + "app_category", "app_content", + "app_extra_info", "dup_traffic_flag", + "tunnel_id_list", "tunnel_endpoint_a_desc", "tunnel_endpoint_b_desc", - "protocol_path", + "ip_protocol", + "decoded_path", "fqdn_category_list", "sent_pkts", "received_pkts", @@ -483,6 +456,7 @@ "mail_attachment_name", "mail_attachment_name_charset", "mail_eml_file", + "mail_starttls_flag", "dns_message_id", "dns_qr", "dns_opcode", @@ -497,8 +471,6 @@ "dns_nscount", "dns_arcount", "dns_qname", - "dns_qtype", - "dns_qclass", "dns_cname", "dns_sub", "dns_rr", @@ -577,13 +549,180 @@ "rdp_certificate_count", "rdp_certificate_permanent", "rdp_encryption_level", - "rdp_encryption_method" + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "internal_port_list", + "external_port_list", + "internal_asn_list", + "external_asn_list", + "security_rule_id", + "monitor_rule_id", + "proxy_rule_id", + "statistics_rule_id", + "shaping_rule_id", + "sc_rule_id" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" }, "details": { - "$ref": "public_schema_info.json#/details" + "general": { + "$ref": "public_schema_info.json#/details/general" + }, + "treatment": { + "$ref": "public_schema_info.json#/details/treatment" + }, + "source": { + "$ref": "public_schema_info.json#/details/source" + }, + "destination": { + "$ref": "public_schema_info.json#/details/destination" + }, + "application": { + "$ref": "public_schema_info.json#/details/application" + }, + "protocol": [ + "ip_protocol", + "decoded_path", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "ftp_account", + "ftp_url", + "ftp_link_type", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "mail_starttls_flag", + "quic_version", + "quic_sni", + "quic_user_agent", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe" + ], + "transmission": { + "$ref": "public_schema_info.json#/details/transmission" + }, + "other": { + "$ref": "public_schema_info.json#/details/other" + } } }, "data_view": { @@ -622,6 +761,9 @@ "RTP": { "$ref": "public_schema_info.json#/decoded_as/RTP" }, + "RDP": { + "$ref": "public_schema_info.json#/decoded_as/RDP" + }, "SSH": { "$ref": "public_schema_info.json#/decoded_as/SSH" }, @@ -643,7 +785,7 @@ "recv_time", "log_id", "flags_identify_info", - "tunnels", + "encapsulation", "app_debug_info", "app_content", "packet_capture_file", @@ -654,7 +796,195 @@ "$ref": "public_schema_info.json#/tunnel_type" }, "measurements": { - "aggregates": { + "aggregated_metric_unit": { + "SUM": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes" + }, + "RATE": { + "sent_pkts": "pps", + "received_pkts": "pps", + "sent_bytes": "Bps", + "received_bytes": "Bps", + "tcp_c2s_lost_bytes": "Bps", + "tcp_s2c_lost_bytes": "Bps", + "tcp_c2s_o3_pkts": "pps", + "tcp_s2c_o3_pkts": "pps", + "tcp_c2s_rtx_pkts": "pps", + "tcp_s2c_rtx_pkts": "pps", + "tcp_c2s_rtx_bytes": "Bps", + "tcp_s2c_rtx_bytes": "Bps", + "http_request_content_length": "Bps", + "http_response_content_length": "Bps" + }, + "BITRATE": { + "sent_bytes": "bps", + "received_bytes": "bps", + "tcp_c2s_lost_bytes": "bps", + "tcp_s2c_lost_bytes": "bps", + "tcp_c2s_rtx_bytes": "bps", + "tcp_s2c_rtx_bytes": "bps", + "http_request_content_length": "bps", + "http_response_content_length": "bps" + }, + "MAX": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MIN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "AVG": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MEDIAN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P95_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P99_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + } + }, + "field_discovery_metric": { "sessions": [ { "fn": "count", @@ -692,7 +1022,153 @@ } ] } - } + }, + "expression_fields": [ + { + "name": "internal_ip_list", + "label": "Internal IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_ip,''), if(bitAnd(flags, 16)=16, server_ip,''))" + } + }, + { + "name": "external_ip_list", + "label": "External IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_ip,''), if(bitAnd(flags, 16)!=16, server_ip,''))" + } + }, + { + "name": "internal_port_list", + "label": "Internal Port List", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_port, null), if(bitAnd(flags, 16)=16, server_port, null))" + } + }, + { + "name": "external_port_list", + "label": "External Port List", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_port, null), if(bitAnd(flags, 16)!=16, server_port, null))" + } + }, + { + "name": "internal_asn_list", + "label": "Internal ASN List", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_asn, null), if(bitAnd(flags, 16)=16, server_asn, null))" + } + }, + { + "name": "external_asn_list", + "label": "External ASN List", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_asn, null), if(bitAnd(flags, 16)!=16, server_asn, null))" + } + }, + { + "name": "security_rule_id", + "label": "Security Rule ID", + "type": "long", + "doc": { + "allow_query": "false", + "expression": "arrayJoin(IF(empty(security_rule_list), arrayPushFront(security_rule_list, null),security_rule_list))" + } + }, + { + "name": "proxy_rule_id", + "label": "Proxy Rule ID", + "type": "long", + "doc": { + "allow_query": "false", + "expression": "arrayJoin(IF(empty(proxy_rule_list), arrayPushFront(proxy_rule_list, null),proxy_rule_list))" + } + }, + { + "name": "monitor_rule_id", + "label": "Monitor Rule ID", + "type": "long", + "doc": { + "allow_query": "false", + "expression": "arrayJoin(IF(empty(monitor_rule_list), arrayPushFront(monitor_rule_list, null),monitor_rule_list))" + } + }, + { + "name": "shaping_rule_id", + "label": "Shaping Rule ID", + "type": "long", + "doc": { + "allow_query": "false", + "expression": "arrayJoin(IF(empty(shaping_rule_list), arrayPushFront(shaping_rule_list, null),shaping_rule_list))" + } + }, + { + "name": "sc_rule_id", + "label": "Service Chaining Rule ID", + "type": "long", + "doc": { + "allow_query": "false", + "expression": "arrayJoin(IF(empty(sc_rule_list), arrayPushFront(sc_rule_list, null),sc_rule_list))" + } + }, + { + "name": "statistics_rule_id", + "label": "Statistics Rule ID", + "type": "long", + "doc": { + "allow_query": "false", + "expression": "arrayJoin(IF(empty(statistics_rule_list), arrayPushFront(statistics_rule_list, null),statistics_rule_list))" + } + } + ] }, "fields": [ { @@ -714,11 +1190,9 @@ "name": "log_id", "label": "Log ID", "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], + "constraints": { + "aggregation_functions": "COUNT" + }, "visibility": "enabled", "ttl": null }, @@ -729,7 +1203,7 @@ "label": "Decoded AS", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -807,11 +1281,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -828,11 +1297,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -843,10 +1307,11 @@ }, { "name": "duration_ms", - "label": "Duration (ms)", + "label": "Duration(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -855,10 +1320,11 @@ }, { "name": "tcp_handshake_latency_ms", - "label": "TCP Handshake Latency (ms)", + "label": "TCP Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -872,11 +1338,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "current_timestamp" - } - ], "visibility": "enabled", "ttl": null }, @@ -892,15 +1353,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "ingestion_time" - }, - { - "function": "get_value", - "appendTo": "recv_time" - } - ], "visibility": "enabled", "ttl": null }, @@ -956,18 +1408,6 @@ "label": "Device Tag", "doc": { "visibility": "hidden", - "format": [ - { - "function": "flattenSpec", - "appendTo": "data_center", - "param": "$.tags[?(@.tag=='data_center')][0].value" - }, - { - "function": "flattenSpec", - "appendTo": "device_group", - "param": "$.tags[?(@.tag=='device_group')][0].value" - } - ], "ttl": null }, "type": "string" @@ -977,7 +1417,7 @@ "label": "Data Center", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -994,7 +1434,7 @@ "label": "Device Group", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -1011,6 +1451,7 @@ "label": "Sled IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, "visibility": "enabled", @@ -1025,22 +1466,30 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/address_type/data" + }, "visibility": "enabled", "ttl": null }, "type": "int" }, { + "name": "direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/direction/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { "name": "vsys_id", "label": "Vsys ID", "doc": { @@ -1089,6 +1538,30 @@ "type": "string" }, { + "name": "c2s_ttl", + "label": "Client-to-Server TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { + "name": "s2c_ttl", + "label": "Server-to-Client TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { "name": "security_rule_list", "label": "Security Rule List", "doc": { @@ -1110,18 +1583,11 @@ "doc": { "visibility": "enabled", "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/security_action/data" }, - "data": [ - { - "code": "Deny", - "value": "Deny" - }, - { - "code": "Allow", - "value": "Allow" - } - ], "ttl": null }, "type": "string" @@ -1159,8 +1625,24 @@ } }, { + "name": "statistics_rule_list", + "label": "Statistics Rule List", + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": null + }, + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + } + }, + { "name": "sc_rsp_raw", - "label": "Service Chaining Rendered Service Path (Raw)", + "label": "Service Chaining Rendered Service Path(Raw)", "doc": { "constraints": { "operator_functions": "has,notEmpty,empty" @@ -1176,7 +1658,7 @@ }, { "name": "sc_rsp_decrypted", - "label": "Service Chaining Rendered Service Path (Decrypted)", + "label": "Service Chaining Rendered Service Path(Decrypted)", "doc": { "constraints": { "operator_functions": "has,notEmpty,empty" @@ -1228,7 +1710,7 @@ "doc": { "visibility": "enabled", "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -1248,20 +1730,12 @@ "name": "proxy_pinning_status", "label": "Proxy Pinning Status", "doc": { - "data": [ - { - "code": "0", - "value": "not pinning" - }, - { - "code": "1", - "value": "pinning" - }, - { - "code": "2", - "value": "maybe pinning" - } - ], + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/proxy_pinning_status/data" + }, "visibility": "enabled", "ttl": null }, @@ -1271,20 +1745,12 @@ "name": "proxy_intercept_status", "label": "Proxy Intercept Status", "doc": { - "data": [ - { - "code": "0", - "value": "passthrough" - }, - { - "code": "1", - "value": "intercept" - }, - { - "code": "2", - "value": "shutdown" - } - ], + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/proxy_intercept_status/data" + }, "visibility": "enabled", "ttl": null }, @@ -1301,7 +1767,7 @@ }, { "name": "proxy_server_side_latency_ms", - "label": "Proxy Server Side Latency", + "label": "Proxy Server-Side Latency(ms)", "doc": { "visibility": "enabled", "ttl": null @@ -1310,7 +1776,7 @@ }, { "name": "proxy_client_side_latency_ms", - "label": "Proxy Client Side Latency", + "label": "Proxy Client-Side Latency(ms)", "doc": { "visibility": "enabled", "ttl": null @@ -1319,7 +1785,7 @@ }, { "name": "proxy_client_side_version", - "label": "Proxy Client Side Version", + "label": "Proxy Client-Side Version", "doc": { "visibility": "enabled", "ttl": null @@ -1328,7 +1794,7 @@ }, { "name": "proxy_server_side_version", - "label": "Proxy Server Side Version", + "label": "Proxy Server-Side Version", "doc": { "visibility": "enabled", "ttl": null @@ -1337,7 +1803,7 @@ }, { "name": "proxy_cert_verify", - "label": "Proxy Cert Verify", + "label": "Proxy Certificate Verify", "doc": { "visibility": "enabled", "ttl": null @@ -1354,8 +1820,8 @@ "type": "string" }, { - "name": "security_mirrored_pkts", - "label": "Security Mirrored Packets", + "name": "monitor_mirrored_pkts", + "label": "Monitor Mirrored Packets", "type": "int", "doc": { "constraints": { @@ -1366,8 +1832,8 @@ } }, { - "name": "security_mirrored_bytes", - "label": "Security Mirrored Bytes", + "name": "monitor_mirrored_bytes", + "label": "Monitor Mirrored Bytes", "type": "int", "doc": { "constraints": { @@ -1382,18 +1848,9 @@ "label": "Client IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "client_asn" - }, - { - "function": "radius_match", - "appendTo": "subscriber_id" - } - ], "visibility": "enabled", "ttl": null }, @@ -1403,6 +1860,10 @@ "name": "client_port", "label": "Client Port", "doc": { + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1419,7 +1880,52 @@ }, { "name": "client_geolocation", - "label": "Client Geolocation", + "label": "Client Geolocation(Deprecated)", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_country", + "label": "Client Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_super_administrative_area", + "label": "Client Super Administrative Area", + "doc": { + "constraints": { + "type": "super_administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_administrative_area", + "label": "Client Administrative Area", + "doc": { + "constraints": { + "type": "administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_sub_administrative_area", + "label": "Client SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1430,6 +1936,9 @@ "name": "client_asn", "label": "Client ASN", "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1485,14 +1994,9 @@ "label": "Server IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "server_asn" - } - ], "visibility": "enabled", "ttl": null }, @@ -1502,6 +2006,10 @@ "name": "server_port", "label": "Server Port", "doc": { + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1518,7 +2026,52 @@ }, { "name": "server_geolocation", - "label": "Server Geolocation", + "label": "Server Geolocation(Deprecated)", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_country", + "label": "Server Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_super_administrative_area", + "label": "Server Super Administrative Area", + "doc": { + "constraints": { + "type": "super_administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_administrative_area", + "label": "Server Administrative Area", + "doc": { + "constraints": { + "type": "administrative_area" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_sub_administrative_area", + "label": "Server SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1529,6 +2082,9 @@ "name": "server_asn", "label": "Server ASN", "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "visibility": "enabled", "ttl": null }, @@ -1538,6 +2094,9 @@ "name": "server_fqdn", "label": "Server FQDN", "doc": { + "constraints": { + "type": "fqdn" + }, "visibility": "enabled", "ttl": null }, @@ -1547,14 +2106,17 @@ "name": "server_domain", "label": "Server Domain", "doc": { + "constraints": { + "type": "domain" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "app_path", - "label": "Application Path", + "name": "app_transition", + "label": "Application Transition", "type": "string", "doc": { "visibility": "enabled", @@ -1565,6 +2127,18 @@ "name": "app", "label": "Application", "doc": { + "constraints": { + "type": "app" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "app_category", + "label": "Application Category", + "doc": { "visibility": "enabled", "ttl": null }, @@ -1574,7 +2148,7 @@ "name": "app_debug_info", "label": "Application Debug Info", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "string" @@ -1589,8 +2163,32 @@ "type": "string" }, { - "name": "protocol_path", - "label": "Protocol Path", + "name": "app_extra_info", + "label": "Application Extra Info", + "doc": { + "visibility": "hidden", + "ttl": null + }, + "type": "string" + }, + { + "name": "ip_protocol", + "label": "IP Protocol", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/ip_protocol/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "decoded_path", + "label": "Decoded Path", "doc": { "visibility": "enabled", "ttl": null @@ -1605,16 +2203,16 @@ "operator_functions": "has,notEmpty,empty" }, "dict_location": { - "path": "/v1/category/dict", - "key": "categoryId", - "value": "categoryName" + "path": "/v1/policy/object?type=fqdn_category", + "key": "category_id", + "value": "category_name" }, "visibility": "enabled", "ttl": null }, "type": { "type": "array", - "items": "int", + "items": "long", "logicalType": "array" } }, @@ -1668,7 +2266,7 @@ }, { "name": "tcp_c2s_ip_fragments", - "label": "TCP C2S IP Fragments ", + "label": "Client-to-Server IP Fragments", "doc": { "constraints": { "type": "decimal" @@ -1680,7 +2278,7 @@ }, { "name": "tcp_s2c_ip_fragments", - "label": "TCP S2C IP Fragments", + "label": "Server-to-Client IP Fragments", "doc": { "constraints": { "type": "decimal" @@ -1692,7 +2290,7 @@ }, { "name": "tcp_c2s_lost_bytes", - "label": "TCP C2S Lost Bytes", + "label": "Client-to-Server Lost Bytes", "doc": { "constraints": { "type": "bytes" @@ -1704,7 +2302,7 @@ }, { "name": "tcp_s2c_lost_bytes", - "label": "TCP S2C Lost Bytes", + "label": "Server-to-Client Lost Bytes", "doc": { "constraints": { "type": "bytes" @@ -1716,7 +2314,7 @@ }, { "name": "tcp_c2s_o3_pkts", - "label": "TCP C2S out-of-order packets", + "label": "Client-to-Server Out-of-Order Packets", "doc": { "constraints": { "type": "decimal" @@ -1728,7 +2326,7 @@ }, { "name": "tcp_s2c_o3_pkts", - "label": "TCP S2C out-of-order packets", + "label": "Server-to-Client Out-of-Order Packets", "doc": { "constraints": { "type": "decimal" @@ -1740,7 +2338,7 @@ }, { "name": "tcp_c2s_rtx_pkts", - "label": "TCP C2S Retransmission Packets", + "label": "Client-to-Server Retransmission Packets", "doc": { "constraints": { "type": "decimal" @@ -1752,7 +2350,7 @@ }, { "name": "tcp_s2c_rtx_pkts", - "label": "TCP S2C Retransmission Packets", + "label": "Server-to-Client Retransmission Packets", "doc": { "constraints": { "type": "decimal" @@ -1764,7 +2362,7 @@ }, { "name": "tcp_c2s_rtx_bytes", - "label": "TCP C2S Retransmission Bytes", + "label": "Client-to-Server Retransmission Bytes", "doc": { "constraints": { "type": "bytes" @@ -1776,7 +2374,7 @@ }, { "name": "tcp_s2c_rtx_bytes", - "label": "TCP S2C Retransmission Bytes", + "label": "Server-to-Client Retransmission Bytes", "doc": { "constraints": { "type": "bytes" @@ -1788,10 +2386,11 @@ }, { "name": "tcp_rtt_ms", - "label": "Round-trip Time", + "label": "Round-trip Time(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -1800,7 +2399,7 @@ }, { "name": "tcp_client_isn", - "label": "TCP Client ISN", + "label": "Client ISN", "doc": { "visibility": "enabled", "ttl": null @@ -1809,7 +2408,7 @@ }, { "name": "tcp_server_isn", - "label": "TCP Server ISN", + "label": "Server ISN", "doc": { "visibility": "enabled", "ttl": null @@ -1866,18 +2465,11 @@ "type": "string" }, { - "name": "tunnels", - "label": "Tunnels", + "name": "encapsulation", + "label": "Encapsulation", "doc": { "visibility": "enabled", - "ttl": null, - "format": [ - { - "function": "gtpc_match", - "appendTo": "imsi,imei,phone_number", - "param": "$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_a2b_teid,$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_b2a_teid" - } - ] + "ttl": null }, "type": "string" }, @@ -1897,8 +2489,21 @@ "type": "int" }, { + "name": "tunnel_id_list", + "label": "Tunnel ID List", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + } + }, + { "name": "tunnel_endpoint_a_desc", - "label": "Tunnel Endpoint A Description", + "label": "Tunnel Endpoint A Description(Deprecated)", "doc": { "visibility": "enabled", "ttl": null @@ -1907,7 +2512,7 @@ }, { "name": "tunnel_endpoint_b_desc", - "label": "Tunnel Endpoint B Description", + "label": "Tunnel Endpoint B Description(Deprecated)", "doc": { "visibility": "enabled", "ttl": null @@ -1918,6 +2523,9 @@ "name": "http_url", "label": "HTTP.URL", "doc": { + "constraints": { + "type": "url" + }, "visibility": "enabled", "ttl": null }, @@ -1952,16 +2560,16 @@ }, { "name": "http_request_content_length", - "label": "HTTP.Request Content Length", + "label": "HTTP.Request Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_request_content_type", - "label": "HTTP.Request Content Type", + "label": "HTTP.Request Content-Type", "doc": { "visibility": "enabled", "ttl": null @@ -1970,12 +2578,12 @@ }, { "name": "http_response_content_length", - "label": "HTTP.Response Content Length", + "label": "HTTP.Response Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_response_content_type", @@ -1990,11 +2598,6 @@ "name": "http_request_body", "label": "HTTP.Request Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -2008,11 +2611,6 @@ "name": "http_response_body", "label": "HTTP.Response Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -2026,7 +2624,7 @@ "name": "http_proxy_flag", "label": "HTTP.Proxy Flag", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "int" @@ -2060,7 +2658,7 @@ }, { "name": "http_user_agent", - "label": "HTTP.User Agent", + "label": "HTTP.User-Agent", "doc": { "visibility": "enabled", "ttl": null @@ -2069,7 +2667,7 @@ }, { "name": "http_set_cookie", - "label": "HTTP.Set Cookie", + "label": "HTTP.Set-Cookie", "doc": { "visibility": "enabled", "ttl": null @@ -2090,21 +2688,26 @@ "label": "HTTP.Status Code", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT", + "type": "http_code" + }, "ttl": null }, "type": "int" }, { "name": "http_response_latency_ms", - "label": "HTTP.Response Latency (ms)", + "label": "HTTP.Response Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "http_action_file_size", @@ -2116,23 +2719,24 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "http_session_duration_ms", - "label": "HTTP.Session Duration (ms)", + "label": "HTTP.Session Duration(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "mail_protocol_type", - "label": "Mail.Protocol Type", + "label": "MAIL.Protocol Type", "doc": { "visibility": "enabled", "ttl": null @@ -2141,8 +2745,11 @@ }, { "name": "mail_account", - "label": "Mail.Account", + "label": "MAIL.Account", "doc": { + "constraints": { + "type": "email" + }, "visibility": "enabled", "ttl": null }, @@ -2150,7 +2757,7 @@ }, { "name": "mail_from_cmd", - "label": "Mail.From CMD", + "label": "MAIL.From CMD", "doc": { "visibility": "enabled", "ttl": null @@ -2159,7 +2766,7 @@ }, { "name": "mail_to_cmd", - "label": "Mail.To CMD", + "label": "MAIL.To CMD", "doc": { "visibility": "enabled", "ttl": null @@ -2168,7 +2775,7 @@ }, { "name": "mail_from", - "label": "Mail.From", + "label": "MAIL.From", "doc": { "constraints": { "type": "email" @@ -2180,7 +2787,7 @@ }, { "name": "mail_password", - "label": "Mail.Password", + "label": "MAIL.Password", "doc": { "visibility": "enabled", "ttl": null @@ -2189,7 +2796,7 @@ }, { "name": "mail_to", - "label": "Mail.To", + "label": "MAIL.To", "doc": { "constraints": { "type": "email" @@ -2201,7 +2808,7 @@ }, { "name": "mail_cc", - "label": "Mail.CC", + "label": "MAIL.CC", "doc": { "visibility": "enabled", "ttl": null @@ -2210,7 +2817,7 @@ }, { "name": "mail_bcc", - "label": "Mail.BCC", + "label": "MAIL.BCC", "doc": { "visibility": "enabled", "ttl": null @@ -2219,14 +2826,8 @@ }, { "name": "mail_subject", - "label": "Mail.Subject", + "label": "MAIL.Subject", "doc": { - "format": [ - { - "function": "decode_of_base64", - "param": "$.mail_subject_charset" - } - ], "visibility": "enabled", "ttl": null }, @@ -2234,7 +2835,7 @@ }, { "name": "mail_subject_charset", - "label": "Mail.Subject Charset", + "label": "MAIL.Subject Charset", "doc": { "visibility": "enabled", "ttl": null @@ -2243,14 +2844,8 @@ }, { "name": "mail_attachment_name", - "label": "Mail.Attachment", + "label": "MAIL.Attachment Name", "doc": { - "format": [ - { - "function": "decode_of_base64", - "param": "$.mail_attachment_name_charset" - } - ], "visibility": "enabled", "ttl": null }, @@ -2258,7 +2853,7 @@ }, { "name": "mail_attachment_name_charset", - "label": "Mail.Attachment Charset", + "label": "MAIL.Attachment Name Charset", "doc": { "visibility": "enabled", "ttl": null @@ -2267,13 +2862,8 @@ }, { "name": "mail_eml_file", - "label": "Mail.EML File", + "label": "MAIL.EML File", "doc": { - "format": [ - { - "function": "combine" - } - ], "constraints": { "type": "file" }, @@ -2283,6 +2873,21 @@ "type": "string" }, { + "name": "mail_starttls_flag", + "label": "MAIL.STARTTLS Flag", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/mail_starttls_flag/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { "name": "dns_message_id", "label": "DNS.Message ID", "doc": { @@ -2298,16 +2903,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "QUERY" - }, - { - "code": "1", - "value": "RESPONSE" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_qr/data" + }, "visibility": "enabled", "ttl": null }, @@ -2318,26 +2916,12 @@ "label": "DNS.OPCODE", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": { + "$ref": "public_schema_info.json#/fields/dns_opcode/data" }, - "data": [ - { - "code": "0", - "value": "QUERY" - }, - { - "code": "1", - "value": "IQUERY" - }, - { - "code": "2", - "value": "STATUS" - }, - { - "code": "5", - "value": "UPDATE" - } - ], "visibility": "enabled", "ttl": null }, @@ -2348,6 +2932,9 @@ "label": "DNS.AA", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -2383,77 +2970,14 @@ "name": "dns_rcode", "label": "DNS.RCODE", "doc": { - "data": [ - { - "code": 0, - "value": "NoError" - }, - { - "code": 1, - "value": "FormErr" - }, - { - "code": 2, - "value": "ServFail" - }, - { - "code": 3, - "value": "NXDomain" - }, - { - "code": 4, - "value": "NotImp" - }, - { - "code": 5, - "value": "Refused" - }, - { - "code": 6, - "value": "YXDomain" - }, - { - "code": 7, - "value": "YXRRSet" - }, - { - "code": 8, - "value": "NXRRSet" - }, - { - "code": 9, - "value": "NotAuth" - }, - { - "code": 10, - "value": "NotZone" - }, - { - "code": 16, - "value": "BADSIG" - }, - { - "code": 17, - "value": "BADKEY" - }, - { - "code": 18, - "value": "BADTIME" - }, - { - "code": 19, - "value": "BADMODE" - }, - { - "code": 20, - "value": "BADNAME" - }, - { - "code": 21, - "value": "BADALG" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_rcode/data" + }, "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -2520,330 +3044,12 @@ "label": "DNS.QTYPE", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": { + "$ref": "public_schema_info.json#/fields/dns_qtype/data" }, - "data": [ - { - "code": "1", - "value": "A" - }, - { - "code": "2", - "value": "NS" - }, - { - "code": "3", - "value": "MD" - }, - { - "code": "4", - "value": "MF" - }, - { - "code": "5", - "value": "CNAME" - }, - { - "code": "6", - "value": "SOA" - }, - { - "code": "7", - "value": "MB" - }, - { - "code": "8", - "value": "MG" - }, - { - "code": "9", - "value": "MR" - }, - { - "code": "10", - "value": "NULL" - }, - { - "code": "11", - "value": "WKS" - }, - { - "code": "12", - "value": "PTR" - }, - { - "code": "13", - "value": "HINFO" - }, - { - "code": "14", - "value": "MINFO" - }, - { - "code": "15", - "value": "MX" - }, - { - "code": "16", - "value": "TXT" - }, - { - "code": "17", - "value": "RP" - }, - { - "code": "18", - "value": "AFSDB" - }, - { - "code": "19", - "value": "X25" - }, - { - "code": "20", - "value": "ISDN" - }, - { - "code": "21", - "value": "RT" - }, - { - "code": "22", - "value": "NSAP" - }, - { - "code": "23", - "value": "NSAP" - }, - { - "code": "24", - "value": "SIG" - }, - { - "code": "25", - "value": "KEY" - }, - { - "code": "26", - "value": "PX" - }, - { - "code": "27", - "value": "GPOS" - }, - { - "code": "28", - "value": "AAAA" - }, - { - "code": "29", - "value": "LOC" - }, - { - "code": "30", - "value": "EID" - }, - { - "code": "31", - "value": "NIMLOC" - }, - { - "code": "32", - "value": "NB" - }, - { - "code": "33", - "value": "SRV" - }, - { - "code": "34", - "value": "ATMA" - }, - { - "code": "35", - "value": "NAPTR" - }, - { - "code": "36", - "value": "KX" - }, - { - "code": "37", - "value": "CERT" - }, - { - "code": "38", - "value": "A6" - }, - { - "code": "39", - "value": "DNAME" - }, - { - "code": "40", - "value": "SINK" - }, - { - "code": "41", - "value": "OPT" - }, - { - "code": "42", - "value": "APL" - }, - { - "code": "43", - "value": "DS" - }, - { - "code": "44", - "value": "SSHFP" - }, - { - "code": "45", - "value": "IPSECKEY" - }, - { - "code": "46", - "value": "RRSIG" - }, - { - "code": "47", - "value": "NSEC" - }, - { - "code": "48", - "value": "DNSKEY" - }, - { - "code": "49", - "value": "DHCID" - }, - { - "code": "50", - "value": "NSEC3" - }, - { - "code": "51", - "value": "NSEC3PARAM" - }, - { - "code": "52", - "value": "TLSA" - }, - { - "code": "53", - "value": "SMIMEA" - }, - { - "code": "55", - "value": "HIP" - }, - { - "code": "59", - "value": "CDS" - }, - { - "code": "60", - "value": "CDNSKEY" - }, - { - "code": "61", - "value": "OPENPGPKEY" - }, - { - "code": "62", - "value": "CSYNC" - }, - { - "code": "63", - "value": "ZONEMD" - }, - { - "code": "64", - "value": "SVCB" - }, - { - "code": "65", - "value": "HTTPS" - }, - { - "code": "99", - "value": "SPF" - }, - { - "code": "100", - "value": "UINFO" - }, - { - "code": "101", - "value": "UID" - }, - { - "code": "102", - "value": "GID" - }, - { - "code": "103", - "value": "UNSPEC" - }, - { - "code": "108", - "value": "EUI48" - }, - { - "code": "109", - "value": "EUI64" - }, - { - "code": "249", - "value": "TKEY" - }, - { - "code": "250", - "value": "TSIG" - }, - { - "code": "251", - "value": "IXFR" - }, - { - "code": "252", - "value": "AXFR" - }, - { - "code": "253", - "value": "MAILB" - }, - { - "code": "254", - "value": "MAILA" - }, - { - "code": "255", - "value": "*" - }, - { - "code": "256", - "value": "URI" - }, - { - "code": "257", - "value": "CAA" - }, - { - "code": "32768", - "value": "TA" - }, - { - "code": "32769", - "value": "DLV" - }, - { - "code": "65521", - "value": "INTEGRITY" - } - ], "visibility": "enabled", "ttl": null }, @@ -2854,6 +3060,10 @@ "label": "DNS.QCLASS", "doc": { "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -2874,16 +3084,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "1", - "value": "DNS" - }, - { - "code": "2", - "value": "DNSSEC" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_sub/data" + }, "visibility": "enabled", "ttl": null }, @@ -2900,12 +3103,13 @@ }, { "name": "dns_response_latency_ms", - "label": "DNS.Response Latency (ms)", + "label": "DNS.Response Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "int" @@ -2948,10 +3152,11 @@ }, { "name": "ssl_handshake_latency_ms", - "label": "SSL.Handshake Latency (ms)", + "label": "SSL.Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -2960,7 +3165,7 @@ }, { "name": "ssl_ja3_hash", - "label": "SSL.JA3 hash", + "label": "SSL.JA3 Hash", "doc": { "visibility": "enabled", "ttl": null @@ -2969,7 +3174,7 @@ }, { "name": "ssl_ja3s_hash", - "label": "SSL.JA3S hash", + "label": "SSL.JA3S Hash", "doc": { "visibility": "enabled", "ttl": null @@ -3004,6 +3209,12 @@ "name": "ssl_esni_flag", "label": "SSL.ESNI Flag", "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/ssl_esni_flag/data" + }, "visibility": "enabled", "ttl": null }, @@ -3013,6 +3224,12 @@ "name": "ssl_ech_flag", "label": " SSL.ECH Flag", "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/ssl_ech_flag/data" + }, "visibility": "enabled", "ttl": null }, @@ -3022,7 +3239,7 @@ "name": "dtls_cookie", "label": "DTLS.Cookie", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" @@ -3031,7 +3248,7 @@ "name": "dtls_version", "label": "DTLS.Version", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" @@ -3049,7 +3266,7 @@ "name": "dtls_san", "label": "DTLS.SAN", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" @@ -3058,62 +3275,63 @@ "name": "dtls_cn", "label": "DTLS.CN", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_handshake_latency_ms", - "label": "DTLS.Handshake Latency (ms)", + "label": "DTLS.Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "int" }, { "name": "dtls_ja3_fingerprint", - "label": "DTLS.JA3", + "label": "DTLS.JA3 Fingerprint", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_ja3_hash", - "label": "DTLS.JA3 hash", + "label": "DTLS.JA3 Hash", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_cert_issuer", - "label": "DTLS.Issuer", + "label": "DTLS.Certificate Issuer", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "dtls_cert_subject", - "label": "DTLS.Subject", + "label": "DTLS.Certificate Subject", "doc": { - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "string" }, { "name": "quic_version", - "label": "Quic.Version", + "label": "QUIC.Version", "doc": { "visibility": "enabled", "ttl": null @@ -3122,7 +3340,7 @@ }, { "name": "quic_sni", - "label": "Quic.SNI", + "label": "QUIC.SNI", "doc": { "visibility": "enabled", "ttl": null @@ -3131,7 +3349,7 @@ }, { "name": "quic_user_agent", - "label": "Quic.User Agent", + "label": "QUIC.User-Agent", "doc": { "visibility": "enabled", "ttl": null @@ -3214,6 +3432,9 @@ "name": "sip_originator_sdp_connect_ip", "label": "SIP.Originator IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -3223,6 +3444,9 @@ "name": "sip_originator_sdp_media_port", "label": "SIP.Originator Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -3250,6 +3474,9 @@ "name": "sip_responder_sdp_connect_ip", "label": "SIP.Responder IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -3259,6 +3486,9 @@ "name": "sip_responder_sdp_media_port", "label": "SIP.Responder Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -3284,7 +3514,7 @@ }, { "name": "sip_duration_s", - "label": "SIP.Duration (s)", + "label": "SIP.Duration(s)", "doc": { "constraints": { "type": "decimal" @@ -3305,121 +3535,14 @@ }, { "name": "rtp_payload_type_c2s", - "label": "RTP.Payload Type (c2s)", + "label": "RTP.Payload Type(Client-to-Server)", "doc": { "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "PCMU" - }, - { - "code": "1", - "value": "1016" - }, - { - "code": "2", - "value": "G721" - }, - { - "code": "3", - "value": "GSM" - }, - { - "code": "4", - "value": "G723" - }, - { - "code": "5", - "value": "DVI4_8000" - }, - { - "code": "6", - "value": "DVI4_16000" - }, - { - "code": "7", - "value": "LPC" - }, - { - "code": "8", - "value": "PCMA" - }, - { - "code": "9", - "value": "G722" - }, - { - "code": "10", - "value": "L16_STEREO" - }, - { - "code": "11", - "value": "L16_MONO" - }, - { - "code": "12", - "value": "QCELP" - }, - { - "code": "13", - "value": "CN" - }, - { - "code": "14", - "value": "MPA" - }, - { - "code": "15", - "value": "G728" - }, - { - "code": "16", - "value": "DVI4_11025" - }, - { - "code": "17", - "value": "DVI4_22050" - }, - { - "code": "18", - "value": "G729" - }, - { - "code": "19", - "value": "CN_OLD" - }, - { - "code": "25", - "value": "CELB" - }, - { - "code": "26", - "value": "JPEG" - }, - { - "code": "28", - "value": "NV" - }, - { - "code": "31", - "value": "H261" - }, - { - "code": "32", - "value": "MPV" - }, - { - "code": "33", - "value": "MP2T" - }, - { - "code": "34", - "value": "H263" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_payload_type_c2s/data" + }, "visibility": "enabled", "ttl": null }, @@ -3427,121 +3550,14 @@ }, { "name": "rtp_payload_type_s2c", - "label": "RTP.Payload Type (s2c)", + "label": "RTP.Payload Type(Server-to-Client)", "doc": { "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "PCMU" - }, - { - "code": "1", - "value": "1016" - }, - { - "code": "2", - "value": "G721" - }, - { - "code": "3", - "value": "GSM" - }, - { - "code": "4", - "value": "G723" - }, - { - "code": "5", - "value": "DVI4_8000" - }, - { - "code": "6", - "value": "DVI4_16000" - }, - { - "code": "7", - "value": "LPC" - }, - { - "code": "8", - "value": "PCMA" - }, - { - "code": "9", - "value": "G722" - }, - { - "code": "10", - "value": "L16_STEREO" - }, - { - "code": "11", - "value": "L16_MONO" - }, - { - "code": "12", - "value": "QCELP" - }, - { - "code": "13", - "value": "CN" - }, - { - "code": "14", - "value": "MPA" - }, - { - "code": "15", - "value": "G728" - }, - { - "code": "16", - "value": "DVI4_11025" - }, - { - "code": "17", - "value": "DVI4_22050" - }, - { - "code": "18", - "value": "G729" - }, - { - "code": "19", - "value": "CN_OLD" - }, - { - "code": "25", - "value": "CELB" - }, - { - "code": "26", - "value": "JPEG" - }, - { - "code": "28", - "value": "NV" - }, - { - "code": "31", - "value": "H261" - }, - { - "code": "32", - "value": "MPV" - }, - { - "code": "33", - "value": "MP2T" - }, - { - "code": "34", - "value": "H263" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_payload_type_s2c/data" + }, "visibility": "enabled", "ttl": null }, @@ -3567,20 +3583,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "unknown" - }, - { - "code": "1", - "value": "c2s" - }, - { - "code": "2", - "value": "s2c" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_originator_dir/data" + }, "visibility": "enabled", "ttl": null }, @@ -3651,7 +3656,7 @@ }, { "name": "ssh_kex_alg", - "label": "SSH. Key Exchange Algorithm", + "label": "SSH.Key Exchange Algorithm", "doc": { "visibility": "enabled", "ttl": null @@ -3786,7 +3791,7 @@ }, { "name": "rdp_desktop_width", - "label": "RDP. Desktop Width", + "label": "RDP.Desktop Width", "doc": { "visibility": "enabled", "ttl": null diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/statistics_rule.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/statistics_rule.json index f71e2b9b..e9083362 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/statistics_rule.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/statistics_rule.json @@ -8,44 +8,37 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", "device_group", - "data_center", - "rule_id", - "template_id", - "chart_id", - "version", - "client_ip_object", - "server_ip_object", - "fqdn_category", "client_ip", "server_ip", + "client_asn", + "server_asn", + "client_country", + "server_country", "server_fqdn", "server_domain", - "application" - ], - "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "rule_id", - "template_id", - "chart_id", - "version", + "application", + "fqdn_category", "client_ip_object", "server_ip_object", - "fqdn_category", - "client_ip", - "server_ip", - "server_fqdn", - "server_domain", - "application", + "c2s_ttl", + "s2c_ttl", + "c2s_link_id", + "s2c_link_id", + "client_port", + "server_port" + ], + "metrics": [ "in_bytes", "out_bytes", "bytes", + "in_pkts", + "out_pkts", + "pkts", "new_c2s_flows", "new_s2c_flows", "new_in_sessions", @@ -53,54 +46,117 @@ "sessions", "new_unestablished_sessions", "syn_pkts", + "c2s_ip_fragments", + "s2c_ip_fragments", + "ip_fragments", "session_identifier_sketch", "client_ip_sketch", "server_ip_sketch", "in_latency_ms_sketch", "out_latency_ms_sketch", - "latency_ms_sketch" + "latency_ms_sketch", + "in_pkt_length_sketch", + "out_pkt_length_sketch", + "pkt_length_sketch" ], "filters": [ - "vsys_id", "device_id", "device_group", "data_center", "rule_id", "template_id", "chart_id", - "version", - "client_ip_object", - "server_ip_object", - "fqdn_category", - "client_ip", - "server_ip", - "server_fqdn", - "server_domain", - "application", - "in_bytes", - "out_bytes", - "bytes", - "new_c2s_flows", - "new_s2c_flows", - "new_in_sessions", - "new_out_sessions", - "sessions", - "new_unestablished_sessions", - "syn_pkts", - "session_identifier_sketch", - "client_ip_sketch", - "server_ip_sketch", - "in_latency_ms_sketch", - "out_latency_ms_sketch", - "latency_ms_sketch" + "version" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 15, - "ingestion_delay": 15 + "ingestion_period": "PT1S", + "compaction_period": "PT1M", + "skip_compaction_offset_from_latest": "P7D", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "in_bytes": "bytes", + "out_bytes": "bytes", + "bytes": "bytes", + "in_pkts": "packets", + "out_pkts": "packets", + "pkts": "packets", + "new_c2s_flows": "flows", + "new_s2c_flows": "flows", + "new_in_sessions": "sessions", + "new_out_sessions": "sessions", + "sessions": "sessions", + "new_unestablished_sessions": "sessions", + "syn_pkts": "packets", + "c2s_ip_fragments": "packets", + "s2c_ip_fragments": "packets", + "ip_fragments": "packets" + }, + "RATE": { + "in_bytes": "Bps", + "out_bytes": "Bps", + "bytes": "Bps", + "in_pkts": "pps", + "out_pkts": "pps", + "pkts": "pps", + "new_c2s_flows": "fps", + "new_s2c_flows": "fps", + "new_in_sessions": "sps", + "new_out_sessions": "sps", + "sessions": "sps", + "new_unestablished_sessions": "sps", + "syn_pkts": "pps", + "c2s_ip_fragments": "pps", + "s2c_ip_fragments": "pps", + "ip_fragments": "pps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps", + "bytes": "bps" + }, + "APPROX_COUNT_DISTINCT_HLLD": { + "session_identifier_sketch": "sessions", + "client_ip_sketch": "short", + "server_ip_sketch": "short" + }, + "MEDIAN_HDR": { + "in_latency_ms_sketch": "ms", + "out_latency_ms_sketch": "ms", + "latency_ms_sketch": "ms", + "in_pkt_length_sketch": "packets", + "out_pkt_length_sketch": "packets", + "pkt_length_sketch": "packets" + }, + "P95_PERCENTILE_HDR": { + "in_latency_ms_sketch": "ms", + "out_latency_ms_sketch": "ms", + "latency_ms_sketch": "ms", + "in_pkt_length_sketch": "packets", + "out_pkt_length_sketch": "packets", + "pkt_length_sketch": "packets" + }, + "P99_PERCENTILE_HDR": { + "in_latency_ms_sketch": "ms", + "out_latency_ms_sketch": "ms", + "latency_ms_sketch": "ms", + "in_pkt_length_sketch": "packets", + "out_pkt_length_sketch": "packets", + "pkt_length_sketch": "packets" + }, + "PERCENTILES_HDR": { + "in_latency_ms_sketch": "ms", + "out_latency_ms_sketch": "ms", + "latency_ms_sketch": "ms", + "in_pkt_length_sketch": "packets", + "out_pkt_length_sketch": "packets", + "pkt_length_sketch": "packets" + } + } } }, "fields": [ @@ -124,7 +180,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -181,7 +237,7 @@ "doc": { "visibility": "enabled" }, - "type": "int" + "type": "long" }, { "name": "chart_id", @@ -189,7 +245,7 @@ "doc": { "visibility": "enabled" }, - "type": "int" + "type": "long" }, { "name": "version", @@ -200,11 +256,11 @@ }, "visibility": "enabled" }, - "type": "string" + "type": "long" }, { "name": "client_ip_object", - "label": "Client IP Object", + "label": "Client IP Matched Objects", "doc": { "visibility": "enabled" }, @@ -212,7 +268,7 @@ }, { "name": "server_ip_object", - "label": "Server IP Object", + "label": "Server IP Matched Objects", "doc": { "visibility": "enabled" }, @@ -220,14 +276,9 @@ }, { "name": "fqdn_category", - "label": "FQDN Category", + "label": "WebSketch Categories", "doc": { - "visibility": "enabled", - "dict_location": { - "path": "/v1/category/dict", - "key": "categoryId", - "value": "categoryName" - } + "visibility": "enabled" }, "type": "string" }, @@ -235,6 +286,9 @@ "name": "client_ip", "label": "Client IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled" }, "type": "string" @@ -243,6 +297,47 @@ "name": "server_ip", "label": "Server IP", "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "client_asn", + "label": "Client ASN", + "doc": { + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "server_asn", + "label": "Server ASN", + "doc": { + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "client_country", + "label": "Client Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "server_country", + "label": "Server Country", + "doc": { + "constraints": { + "type": "country" + }, "visibility": "enabled" }, "type": "string" @@ -251,6 +346,9 @@ "name": "server_fqdn", "label": "Server FQDN", "doc": { + "constraints": { + "type": "fqdn" + }, "visibility": "enabled" }, "type": "string" @@ -259,6 +357,9 @@ "name": "server_domain", "label": "Server Domain", "doc": { + "constraints": { + "type": "domain" + }, "visibility": "enabled" }, "type": "string" @@ -267,16 +368,68 @@ "name": "application", "label": "Application", "doc": { + "constraints": { + "type": "app" + }, "visibility": "enabled" }, "type": "string" }, { + "name": "c2s_ttl", + "label": "Client-to-Server TTL", + "doc": { + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "s2c_ttl", + "label": "Server-to-Client TTL", + "doc": { + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "c2s_link_id", + "label": "Client-to-Server Link ID", + "doc": { + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "s2c_link_id", + "label": "Server-to-Client Link ID", + "doc": { + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "client_port", + "label": "Client Port", + "doc": { + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "server_port", + "label": "Server Port", + "doc": { + "visibility": "enabled" + }, + "type": "long" + }, + { "name": "in_bytes", "label": "Incoming Bytes", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" }, "visibility": "enabled" }, @@ -287,7 +440,8 @@ "label": "Outgoing Bytes", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" }, "visibility": "enabled" }, @@ -298,7 +452,44 @@ "label": "Bytes", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "out_pkts", + "label": "Outgoing Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -306,10 +497,11 @@ }, { "name": "new_c2s_flows", - "label": "New C2S Flows", + "label": "New Client-to-Server Flows", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -317,10 +509,11 @@ }, { "name": "new_s2c_flows", - "label": "New S2C Flows", + "label": "New Server-to-Client Flows", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -331,7 +524,8 @@ "label": "New Incoming Sessions", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -342,7 +536,8 @@ "label": "New Outgoing Sessions", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -353,7 +548,8 @@ "label": "Sessions", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -364,7 +560,8 @@ "label": "New Unestablished Sessions", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -375,7 +572,44 @@ "label": "SYN Packets", "doc": { "constraints": { - "metric_type": "counter" + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "c2s_ip_fragments", + "label": "Client-to-Server IP Fragments", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "s2c_ip_fragments", + "label": "Server-to-Client IP Fragments", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "ip_fragments", + "label": "IP Fragments", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" }, "visibility": "enabled" }, @@ -383,11 +617,12 @@ }, { "name": "session_identifier_sketch", - "label": "Session Identifier Sketch", + "label": "Session IDs", "doc": { "constraints": { "type": "HLLDSketchMerge", - "metric_type": "gauge" + "metric_type": "gauge", + "aggregation_functions": "APPROX_COUNT_DISTINCT_HLLD" }, "visibility": "enabled" }, @@ -395,11 +630,12 @@ }, { "name": "client_ip_sketch", - "label": "Client IP Sketch", + "label": "Client IPs", "doc": { "constraints": { "type": "HLLDSketchMerge", - "metric_type": "gauge" + "metric_type": "gauge", + "aggregation_functions": "APPROX_COUNT_DISTINCT_HLLD" }, "visibility": "enabled" }, @@ -407,11 +643,12 @@ }, { "name": "server_ip_sketch", - "label": "Server IP Sketch", + "label": "Server IPs", "doc": { "constraints": { "type": "HLLDSketchMerge", - "metric_type": "gauge" + "metric_type": "gauge", + "aggregation_functions": "APPROX_COUNT_DISTINCT_HLLD" }, "visibility": "enabled" }, @@ -419,11 +656,12 @@ }, { "name": "in_latency_ms_sketch", - "label": "Incoming Latency(ms) Sketch", + "label": "Incoming Latency", "doc": { "constraints": { "type": "HdrHistogramSketchMerge", - "metric_type": "histogram" + "metric_type": "histogram", + "aggregation_functions": "PERCENTILES_HDR, MEDIAN_HDR, P95_PERCENTILE_HDR,P99_PERCENTILE_HDR" }, "visibility": "enabled" }, @@ -431,11 +669,12 @@ }, { "name": "out_latency_ms_sketch", - "label": "Outgoing Latency(ms) Sketch", + "label": "Outgoing Latency", "doc": { "constraints": { "type": "HdrHistogramSketchMerge", - "metric_type": "histogram" + "metric_type": "histogram", + "aggregation_functions": "PERCENTILES_HDR, MEDIAN_HDR, P95_PERCENTILE_HDR,P99_PERCENTILE_HDR" }, "visibility": "enabled" }, @@ -443,11 +682,51 @@ }, { "name": "latency_ms_sketch", - "label": "Latency(ms) Sketch", + "label": "Latency", + "doc": { + "constraints": { + "type": "HdrHistogramSketchMerge", + "metric_type": "histogram", + "aggregation_functions": "PERCENTILES_HDR, MEDIAN_HDR, P95_PERCENTILE_HDR,P99_PERCENTILE_HDR " + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "in_pkt_length_sketch", + "label": "Incoming Packet Length", + "doc": { + "constraints": { + "type": "HdrHistogramSketchMerge", + "metric_type": "histogram", + "aggregation_functions": "PERCENTILES_HDR, MEDIAN_HDR, P95_PERCENTILE_HDR,P99_PERCENTILE_HDR " + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "out_pkt_length_sketch", + "label": "Outgoing Packet Length", + "doc": { + "constraints": { + "type": "HdrHistogramSketchMerge", + "metric_type": "histogram", + "aggregation_functions": "PERCENTILES_HDR, MEDIAN_HDR, P95_PERCENTILE_HDR,P99_PERCENTILE_HDR " + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "pkt_length_sketch", + "label": "Packet Length", "doc": { "constraints": { "type": "HdrHistogramSketchMerge", - "metric_type": "histogram" + "metric_type": "histogram", + "aggregation_functions": "PERCENTILES_HDR, MEDIAN_HDR, P95_PERCENTILE_HDR,P99_PERCENTILE_HDR " }, "visibility": "enabled" }, diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/statistics_rule_hits.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/statistics_rule_hits.json index dbc707ad..ad9349b7 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/statistics_rule_hits.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/statistics_rule_hits.json @@ -8,19 +8,15 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", + "rule_id", "device_group", - "data_center", - "rule_id" + "data_center" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "rule_id", "hit_count", "in_bytes", "out_bytes", @@ -28,24 +24,37 @@ "out_pkts" ], "filters": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "rule_id", - "hit_count", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" + "device_group", + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 1, - "ingestion_delay": 15 + "ingestion_period": "PT1S", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "hit_count": "sessions", + "in_bytes": "bytes", + "out_bytes": "bytes", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "hit_count": "sps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -109,7 +118,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -121,12 +130,16 @@ }, "visibility": "enabled" }, - "type": "int" + "type": "long" }, { "name": "hit_count", "label": "Hit Count", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -135,6 +148,10 @@ "name": "in_bytes", "label": "Incoming Bytes", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -143,6 +160,10 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" }, "type": "long" @@ -151,6 +172,10 @@ "name": "in_pkts", "label": "Incoming Packets", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -159,6 +184,10 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/sys_packet_capture_event.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/sys_packet_capture_event.json deleted file mode 100644 index 7f460684..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/sys_packet_capture_event.json +++ /dev/null @@ -1,1251 +0,0 @@ -{ - "type": "record", - "name": "sys_packet_capture_event", - "namespace": "tsg_galaxy_v3", - "doc": { - "primary_key": "common_log_id", - "partition_key": "common_recv_time", - "index_key": { - "$ref": "public_schema_info_2311.json#/sys_packet_capture_event/index_key" - } - }, - "fields": [ - { - "name": "common_recv_time", - "type": { - "type": "long", - "logicalType": "unix_timestamp" - }, - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "format": [ - { - "function": "current_timestamp" - } - ], - "visibility": "enabled" - }, - "label": "Receive Time" - }, - { - "name": "common_log_id", - "type": "long", - "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], - "visibility": "enabled" - }, - "label": "Log ID" - }, - { - "name": "common_policy_id", - "type": "long", - "doc": { - "visibility": "hidden" - }, - "label": "Policy ID" - }, - { - "name": "common_subscriber_id", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Subscriber ID" - }, - { - "name": "common_imei", - "type": "string", - "doc": { - "visibility": "disabled" - }, - "label": "IMEI" - }, - { - "name": "common_imsi", - "type": "string", - "doc": { - "visibility": "disabled" - }, - "label": "IMSI" - }, - { - "name": "common_phone_number", - "type": "string", - "doc": { - "visibility": "disabled" - }, - "label": "Phone Number" - }, - { - "name": "common_in_src_mac", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Incoming Source MAC" - }, - { - "name": "common_out_src_mac", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Outgoing Source MAC" - }, - { - "name": "common_client_ip", - "type": "string", - "doc": { - "constraints": { - "type": "ip" - }, - "visibility": "enabled" - }, - "label": "Client IP" - }, - { - "name": "common_internal_ip", - "type": "string", - "doc": { - "constraints": { - "type": "ip" - }, - "visibility": "enabled" - }, - "label": "Internal IP" - }, - { - "name": "common_client_port", - "type": "int", - "doc": { - "visibility": "enabled" - }, - "label": "Client Port" - }, - { - "name": "common_client_os_name", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Client OS" - }, - { - "name": "common_l4_protocol", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "L4 Protocol" - }, - { - "name": "common_address_type", - "type": "int", - "doc": { - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], - "visibility": "enabled" - }, - "label": "Address Type" - }, - { - "name": "common_server_ip", - "type": "string", - "doc": { - "constraints": { - "type": "ip" - }, - "visibility": "enabled" - }, - "label": "Server IP" - }, - { - "name": "common_server_port", - "type": "int", - "doc": { - "visibility": "enabled" - }, - "label": "Server Port" - }, - { - "name": "common_server_os_name", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Server OS" - }, - { - "name": "common_external_ip", - "type": "string", - "doc": { - "constraints": { - "type": "ip" - }, - "visibility": "enabled" - }, - "label": "External IP" - }, - { - "name": "common_action", - "type": "int", - "doc": { - "data": [ - { - "code": "0", - "value": "None" - }, - { - "code": "1", - "value": "Monitor" - }, - { - "code": "2", - "value": "Intercept" - }, - { - "code": "16", - "value": "Deny" - }, - { - "code": "96", - "value": "Allow" - }, - { - "code": "128", - "value": "Allow(Deprecated)" - } - ], - "visibility": "enabled" - }, - "label": "Action" - }, - { - "name": "common_direction", - "type": "int", - "doc": { - "data": [ - { - "code": "69", - "value": "outbound" - }, - { - "code": "73", - "value": "inbound" - } - ], - "visibility": "hidden" - }, - "label": "Direction" - }, - { - "name": "common_entrance_id", - "type": "int", - "doc": { - "visibility": "disabled" - }, - "label": "Entrance ID" - }, - { - "name": "common_sled_ip", - "type": "string", - "doc": { - "constraints": { - "type": "ip" - }, - "visibility": "enabled" - }, - "label": "Sled IP" - }, - { - "name": "common_client_location", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Client Location" - }, - { - "name": "common_client_asn", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Client ASN" - }, - { - "name": "common_server_location", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Server Location" - }, - { - "name": "common_server_asn", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Server ASN" - }, - { - "name": "common_server_fqdn", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Server FQDN" - }, - { - "name": "common_server_domain", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Server Domain" - }, - { - "name": "common_in_dest_mac", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Incoming Destination MAC" - }, - { - "name": "common_out_dest_mac", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Outgoing Destination MAC" - }, - { - "name": "common_sessions", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled" - }, - "label": "Sessions" - }, - { - "name": "common_c2s_pkt_num", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled" - }, - "label": "Packets Sent" - }, - { - "name": "common_s2c_pkt_num", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled" - }, - "label": "Packets Received" - }, - { - "name": "common_c2s_byte_num", - "type": "long", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled" - }, - "label": "Bytes Sent" - }, - { - "name": "common_s2c_byte_num", - "type": "long", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled" - }, - "label": "Bytes Received" - }, - { - "name": "common_c2s_pkt_diff", - "label": "Packets Sent (Delta)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - }, - "type": "long" - }, - { - "name": "common_s2c_pkt_diff", - "label": "Packets Received (Delta)", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - }, - "type": "long" - }, - { - "name": "common_c2s_byte_diff", - "label": "Bytes Sent (Delta)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden" - }, - "type": "long" - }, - { - "name": "common_s2c_byte_diff", - "label": "Bytes Received (Delta)", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden" - }, - "type": "long" - }, - { - "name": "common_service", - "type": "int", - "doc": { - "visibility": "disabled" - }, - "label": "Service" - }, - { - "name": "common_schema_type", - "type": "string", - "doc": { - "data": [ - { - "code": "BASE", - "value": "BASE" - }, - { - "code": "HTTP", - "value": "HTTP" - }, - { - "code": "MAIL", - "value": "MAIL" - }, - { - "code": "DNS", - "value": "DNS" - }, - { - "code": "SSL", - "value": "SSL" - }, - { - "code": "FTP", - "value": "FTP" - } - ], - "visibility": "hidden" - }, - "label": "Schema Type" - }, - { - "name": "common_vsys_id", - "label": "Vsys ID", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "int", - "default": 1 - }, - { - "name": "common_t_vsys_id", - "label": "Traffic Vsys ID", - "doc": { - "allow_query": "false", - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "common_flags", - "label": "Flags", - "doc": { - "constraints": { - "type": "bit", - "operator_functions": "=,!=,bitAnd" - }, - "data": { - "$ref": "public_schema_info_2311.json#/fields/common_flags/data" - }, - "allow_query": "false", - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "long", - "logicalType": "bit" - } - }, - { - "name": "common_flags_identify_info", - "label": "Flags Identify Info", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_user_tags", - "type": "string", - "doc": { - "visibility": "disabled" - }, - "label": "User Tags" - }, - { - "name": "common_sub_action", - "type": "string", - "doc": { - "data": [ - { - "code": "allow", - "value": "Allow" - }, - { - "code": "deny", - "value": "Deny" - }, - { - "code": "monitor", - "value": "Monitor" - }, - { - "code": "replace", - "value": "Replace" - }, - { - "code": "redirect", - "value": "Redirect" - }, - { - "code": "insert", - "value": "Insert" - }, - { - "code": "hijack", - "value": "Hijack" - } - ], - "visibility": "hidden" - }, - "label": "Sub Action" - }, - { - "name": "common_user_region", - "type": "string", - "doc": { - "visibility": "hidden" - }, - "label": "User Region" - }, - { - "name": "common_shaping_rule_ids", - "label": "Shaping Rule IDs", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "int", - "logicalType": "array" - } - }, - { - "name": "common_device_id", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Device ID" - }, - { - "name": "common_out_link_id", - "label": "Outgoing Link ID", - "doc": { - "visibility": "hidden" - }, - "type": "int" - }, - { - "name": "common_in_link_id", - "label": "Incoming Link ID", - "doc": { - "visibility": "hidden" - }, - "type": "int" - }, - { - "name": "common_isp", - "type": "string", - "doc": { - "visibility": "disabled" - }, - "label": "ISP" - }, - { - "name": "common_device_tag", - "type": "string", - "doc": { - "visibility": "hidden", - "format": [ - { - "function": "flattenSpec", - "appendTo": "common_data_center", - "param": "$.tags[?(@.tag=='data_center')][0].value" - }, - { - "function": "flattenSpec", - "appendTo": "common_device_group", - "param": "$.tags[?(@.tag=='device_group')][0].value" - } - ] - }, - "label": "Device Tag" - }, - { - "name": "common_data_center", - "label": "Data Center", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", - "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" - }, - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "common_device_group", - "label": "Device Group", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", - "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" - }, - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "common_app_behavior", - "label": "Application Behavior", - "doc": { - "visibility": "hidden" - }, - "type": "string" - }, - { - "name": "common_encapsulation", - "type": "int", - "doc": { - "data": [ - { - "code": "0", - "value": "Ethernet" - }, - { - "code": "8", - "value": "PPP" - }, - { - "code": "12", - "value": "CiscoHDLC" - } - ], - "visibility": "enabled" - }, - "label": "Encapsulation" - }, - { - "name": "common_app_label", - "type": "string", - "doc": { - "visibility": "disabled" - }, - "label": "Application Label" - }, - { - "name": "common_tunnels", - "type": "string", - "doc": { - "visibility": "hidden", - "format": [ - { - "function": "gtpc_match", - "appendTo": "common_imsi,common_imei,common_phone_number", - "param": "$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_a2b_teid,$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_b2a_teid" - } - ] - }, - "label": "Tunnels" - }, - { - "name": "common_protocol_label", - "type": "string", - "doc": { - "visibility": "hidden" - }, - "label": "Protocol Label" - }, - { - "name": "common_app_id", - "type": "string", - "label": "Application ID", - "doc": { - "visibility": "hidden" - } - }, - { - "name": "common_app_full_path", - "label": "Application Full Path", - "type": "string", - "doc": { - "visibility": "hidden" - } - }, - { - "name": "common_userdefine_app_name", - "label": "User Define App Name", - "type": "string", - "doc": { - "visibility": "hidden" - } - }, - { - "name": "common_app_identify_info", - "label": "App Identity Info", - "doc": { - "visibility": "hidden" - }, - "type": "string" - }, - { - "name": "common_app_surrogate_id", - "type": "string", - "label": "Surrogate ID", - "doc": { - "visibility": "hidden" - } - }, - { - "name": "common_l7_protocol", - "type": "string", - "doc": { - "visibility": "hidden" - }, - "label": "L7 Protocol" - }, - { - "name": "common_service_category", - "label": "FQDN Category", - "doc": { - "visibility": "enabled" - }, - "type": { - "type": "array", - "items": "int", - "logicalType": "array" - } - }, - { - "name": "common_start_time", - "type": { - "type": "long", - "logicalType": "unix_timestamp" - }, - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "hidden" - }, - "label": "Start Time" - }, - { - "name": "common_end_time", - "type": { - "type": "long", - "logicalType": "unix_timestamp" - }, - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "visibility": "hidden" - }, - "label": "End Time" - }, - { - "name": "common_start_timestamp_ms", - "label": "Start Time", - "doc": { - "constraints": { - "operator_functions": "=,!=,>,<,>=,<=", - "type": "datetime64" - }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], - "visibility": "enabled" - }, - "type": { - "type": "string", - "logicalType": "datetime64" - } - }, - { - "name": "common_end_timestamp_ms", - "label": "End Time", - "doc": { - "constraints": { - "operator_functions": "=,!=,>,<,>=,<=", - "type": "datetime64" - }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], - "visibility": "enabled" - }, - "type": { - "type": "string", - "logicalType": "datetime64" - } - }, - { - "name": "common_establish_latency_ms", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - }, - "label": "TCP Handshake Latency (ms)" - }, - { - "name": "common_con_duration_ms", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - }, - "label": "Duration (ms)" - }, - { - "name": "common_stream_dir", - "type": "int", - "doc": { - "data": [ - { - "code": "1", - "value": "c2s" - }, - { - "code": "2", - "value": "s2c" - }, - { - "code": "3", - "value": "double" - } - ], - "visibility": "enabled" - }, - "label": "Stream Direction(Deprecated)" - }, - { - "name": "common_address_list", - "type": "string", - "doc": { - "visibility": "disabled" - }, - "label": "Address List" - }, - { - "name": "common_has_dup_traffic", - "type": "int", - "doc": { - "data": [ - { - "code": "0", - "value": "No" - }, - { - "code": "1", - "value": "Yes" - } - ], - "visibility": "hidden" - }, - "label": "Duplication Traffic" - }, - { - "name": "common_stream_error", - "type": "string", - "doc": { - "visibility": "hidden" - }, - "label": "Stream Error" - }, - { - "name": "common_stream_trace_id", - "type": "long", - "doc": { - "visibility": "enabled" - }, - "label": "Session ID" - }, - { - "name": "common_link_info_c2s", - "type": "string", - "doc": { - "visibility": "hidden" - }, - "label": "Link Info (c2s)" - }, - { - "name": "common_link_info_s2c", - "type": "string", - "doc": { - "visibility": "hidden" - }, - "label": "Link Info (s2c)" - }, - { - "name": "common_packet_capture_file", - "label": "Packet Capture File", - "doc": { - "visibility": "hidden", - "constraints": { - "type": "file" - } - }, - "type": "string" - }, - { - "name": "common_tunnel_endpoint_a_desc", - "label": "Tunnel Endpoint A Description", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_tunnel_endpoint_b_desc", - "label": "Tunnel Endpoint B Description", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "common_c2s_ipfrag_num", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - }, - "label": "Fragmentation Packets (c2s)" - }, - { - "name": "common_s2c_ipfrag_num", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - }, - "label": "Fragmentation Packets (s2c)" - }, - { - "name": "common_c2s_tcp_lostlen", - "type": "long", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden" - }, - "label": "Sequence Gap Loss (c2s)" - }, - { - "name": "common_s2c_tcp_lostlen", - "type": "long", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden" - }, - "label": "Sequence Gap Loss (s2c)" - }, - { - "name": "common_c2s_tcp_unorder_num", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - }, - "label": "Unordered Packets (c2s)" - }, - { - "name": "common_s2c_tcp_unorder_num", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - }, - "label": "Unordered Packets (s2c)" - }, - { - "name": "common_c2s_pkt_retrans", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled" - }, - "label": "Packet Retransmission (c2s)" - }, - { - "name": "common_s2c_pkt_retrans", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled" - }, - "label": "Packet Retransmission (s2c)" - }, - { - "name": "common_c2s_byte_retrans", - "type": "long", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled" - }, - "label": "Byte Retransmission (c2s)" - }, - { - "name": "common_s2c_byte_retrans", - "type": "long", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled" - }, - "label": "Byte Retransmission (s2c)" - }, - { - "name": "common_tcp_client_isn", - "label": "TCP Client ISN", - "doc": { - "visibility": "disabled" - }, - "type": "long" - }, - { - "name": "common_tcp_server_isn", - "label": "TCP Server ISN", - "doc": { - "visibility": "disabled" - }, - "type": "long" - }, - { - "name": "common_first_ttl", - "type": "int", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - }, - "label": "First TTL" - }, - { - "name": "common_processing_time", - "type": { - "type": "long", - "logicalType": "unix_timestamp" - }, - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "format": [ - { - "function": "current_timestamp" - } - ], - "visibility": "enabled" - }, - "label": "Processing Time" - }, - { - "name": "common_ingestion_time", - "label": "Ingestion Time", - "doc": { - "constraints": { - "type": "unix_timestamp" - }, - "format": [ - { - "function": "ingestion_time" - } - ], - "visibility": "enabled" - }, - "type": { - "type": "long", - "logicalType": "unix_timestamp" - } - }, - { - "name": "common_mirrored_pkts", - "label": "Mirrored Packets", - "type": "long", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "hidden" - } - }, - { - "name": "common_mirrored_bytes", - "label": "Mirrored Bytes", - "type": "long", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "hidden" - } - }, - { - "name": "nic_name", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Nic Name" - }, - { - "name": "origin_source_mac", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Origin Source Mac" - }, - { - "name": "origin_dest_mac", - "type": "string", - "doc": { - "visibility": "enabled" - }, - "label": "Origin Dest Mac" - }, - { - "name": "packet_url", - "type": "string", - "doc": { - "constraints": { - "type": "file" - }, - "visibility": "enabled" - }, - "label": "Packet URL" - }, - { - "name": "pcap_storage_task_id", - "type": "int", - "doc": { - "visibility": "enabled" - }, - "label": "Task ID" - }, - { - "name": "pcap_storage_duration", - "type": "int", - "doc": { - "visibility": "enabled" - }, - "label": "Duration" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/sys_storage_log.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/sys_storage_log.json deleted file mode 100644 index 7d498a34..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/sys_storage_log.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "type": "record", - "name": "sys_storage_log", - "namespace": "druid", - "doc": { - "partition_key": "__time", - "functions": { - "$ref": "public_schema_info.json#/functions" - }, - "schema_query": { - "dimensions": [ - "log_type", - "data_center" - ], - "metrics": [ - "log_type", - "data_center", - "used_size", - "max_size", - "aggregate_size", - "last_storage", - "__time" - ], - "filters": [ - "log_type", - "data_center", - "used_size", - "max_size", - "aggregate_size", - "last_storage", - "__time" - ], - "references": { - "$ref": "public_schema_info.json#/schema_query/references" - } - } - }, - "fields": [ - { - "name": "__time", - "label": "Time", - "type": { - "type": "string", - "logicalType": "timestamp" - }, - "doc": { - "constraints": { - "type": "timestamp" - }, - "visibility": "enabled" - } - }, - { - "name": "data_center", - "label": "Data Center", - "type": "string", - "doc": { - "constraints": { - "operator_functions": "=,in" - }, - "data": { - "$ref": "device_tag.json#", - "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", - "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" - }, - "visibility": "enabled" - } - }, - { - "name": "log_type", - "label": "Log Type", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "max_size", - "label": "Max Size", - "doc": { - "visibility": "enabled" - }, - "type": "long" - }, - { - "name": "used_size", - "label": "Used Size", - "doc": { - "visibility": "enabled" - }, - "type": "long" - }, - { - "name": "aggregate_size", - "label": "Aggregate Size", - "doc": { - "visibility": "enabled" - }, - "type": "long" - }, - { - "name": "last_storage", - "label": "Last Storage", - "doc": { - "visibility": "enabled" - }, - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_client_countries.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_client_countries.json new file mode 100644 index 00000000..cb504d82 --- /dev/null +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_client_countries.json @@ -0,0 +1,250 @@ +{ + "type": "record", + "name": "top_client_countries", + "namespace": "druid", + "doc": { + "partition_key": "__time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "time": [ + "__time" + ], + "dimensions": [ + "client_country" + ], + "metrics": [ + "sessions", + "bytes", + "in_bytes", + "out_bytes", + "pkts", + "in_pkts", + "out_pkts" + ], + "filters": [ + "rank_by", + "device_group", + "data_center" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + } + }, + "measurements": { + "ingestion_period": "PT1M", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "bytes": "bytes", + "in_bytes": "bytes", + "out_bytes": "bytes", + "pkts": "packets", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "bytes": "Bps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "pkts": "pps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "bytes": "bps", + "in_bytes": "bps", + "out_bytes": "bps" + } + } + } + }, + "fields": [ + { + "name": "__time", + "label": "Time", + "type": { + "type": "string", + "logicalType": "timestamp" + }, + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "enabled" + } + }, + { + "name": "vsys_id", + "label": "Vsys ID", + "doc": { + "visibility": "enabled" + }, + "type": "long", + "default": 1 + }, + { + "name": "device_id", + "label": "Device ID", + "doc": { + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "device_group", + "label": "Device Group", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "visibility": "enabled" + } + }, + { + "name": "data_center", + "label": "Data Center", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "visibility": "enabled" + } + }, + { + "name": "rank_by", + "label": "Rank", + "doc": { + "constraints": { + "operator_functions": "=" + }, + "data": [ + { + "code": "bytes", + "value": "Bytes" + }, + { + "code": "sessions", + "value": "Sessions" + }, + { + "code": "packets", + "value": "Packets" + } + ], + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "client_country", + "label": "Client Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "sessions", + "label": "Sessions", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "bytes", + "label": "Bytes", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "in_bytes", + "label": "Incoming Bytes", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "out_bytes", + "label": "Outgoing Bytes", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "out_pkts", + "label": "Outgoing Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_client_ips.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_client_ips.json index 0ee3563c..e6bed2fe 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_client_ips.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_client_ips.json @@ -8,44 +8,58 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "client_ip" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "client_ip", "sessions", + "bytes", "in_bytes", "out_bytes", + "pkts", "in_pkts", "out_pkts" ], "filters": [ - "vsys_id", - "device_id", + "rank_by", "device_group", - "data_center", - "client_ip", - "sessions", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 300, - "ingestion_delay": 15 + "ingestion_period": "PT1M", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "bytes": "bytes", + "in_bytes": "bytes", + "out_bytes": "bytes", + "pkts": "packets", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "bytes": "Bps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "pkts": "pps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "bytes": "bps", + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -69,7 +83,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -113,17 +127,37 @@ } }, { - "name": "client_ip", - "label": "Client IP", + "name": "rank_by", + "label": "Rank", "doc": { + "constraints": { + "operator_functions": "=" + }, + "data": [ + { + "code": "bytes", + "value": "Bytes" + }, + { + "code": "sessions", + "value": "Sessions" + }, + { + "code": "packets", + "value": "Packets" + } + ], "visibility": "enabled" }, "type": "string" }, { - "name": "metric", - "label": "Metric", + "name": "client_ip", + "label": "Client IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled" }, "type": "string" @@ -132,23 +166,35 @@ "name": "sessions", "label": "Sessions", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, { - "name": "in_bytes", - "label": "Incoming Bytes", + "name": "bytes", + "label": "Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, { - "name": "in_pkts", - "label": "Incoming Packets", + "name": "in_bytes", + "label": "Incoming Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, @@ -156,7 +202,35 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, @@ -164,7 +238,11 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" } diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_external_ips.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_external_ips.json index 8cf58ee8..3ef25511 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_external_ips.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_external_ips.json @@ -8,44 +8,58 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "external_ip" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "external_ip", "sessions", + "bytes", "in_bytes", "out_bytes", + "pkts", "in_pkts", "out_pkts" ], "filters": [ - "vsys_id", - "device_id", + "rank_by", "device_group", - "data_center", - "external_ip", - "sessions", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 300, - "ingestion_delay": 15 + "ingestion_period": "PT1M", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "bytes": "bytes", + "in_bytes": "bytes", + "out_bytes": "bytes", + "pkts": "packets", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "bytes": "Bps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "pkts": "pps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "bytes": "bps", + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -69,7 +83,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -113,16 +127,33 @@ } }, { - "name": "external_ip", - "label": "External IP", + "name": "rank_by", + "label": "Rank", "doc": { + "constraints": { + "operator_functions": "=" + }, + "data": [ + { + "code": "bytes", + "value": "Bytes" + }, + { + "code": "sessions", + "value": "Sessions" + }, + { + "code": "packets", + "value": "Packets" + } + ], "visibility": "enabled" }, "type": "string" }, { - "name": "metric", - "label": "Metric", + "name": "external_ip", + "label": "External IP", "doc": { "visibility": "enabled" }, @@ -132,23 +163,35 @@ "name": "sessions", "label": "Sessions", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, { - "name": "in_bytes", - "label": "Incoming Bytes", + "name": "bytes", + "label": "Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, { - "name": "in_pkts", - "label": "Incoming Packets", + "name": "in_bytes", + "label": "Incoming Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, @@ -156,7 +199,35 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, @@ -164,7 +235,11 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" } diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_internal_ips.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_internal_ips.json index 27cd4aa6..d851d150 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_internal_ips.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_internal_ips.json @@ -8,44 +8,58 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "internal_ip" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "internal_ip", "sessions", + "bytes", "in_bytes", "out_bytes", + "pkts", "in_pkts", "out_pkts" ], "filters": [ - "vsys_id", - "device_id", + "rank_by", "device_group", - "data_center", - "internal_ip", - "sessions", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 300, - "ingestion_delay": 15 + "ingestion_period": "PT1M", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "bytes": "bytes", + "in_bytes": "bytes", + "out_bytes": "bytes", + "pkts": "packets", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "bytes": "Bps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "pkts": "pps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "bytes": "bps", + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -69,7 +83,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -113,17 +127,37 @@ } }, { - "name": "internal_ip", - "label": "Internal IP", + "name": "rank_by", + "label": "Rank", "doc": { + "constraints": { + "operator_functions": "=" + }, + "data": [ + { + "code": "bytes", + "value": "Bytes" + }, + { + "code": "sessions", + "value": "Sessions" + }, + { + "code": "packets", + "value": "Packets" + } + ], "visibility": "enabled" }, "type": "string" }, { - "name": "metric", - "label": "Metric", + "name": "internal_ip", + "label": "Internal IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled" }, "type": "string" @@ -132,23 +166,35 @@ "name": "sessions", "label": "Sessions", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, { - "name": "in_bytes", - "label": "Incoming Bytes", + "name": "bytes", + "label": "Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, { - "name": "in_pkts", - "label": "Incoming Packets", + "name": "in_bytes", + "label": "Incoming Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, @@ -156,7 +202,35 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, @@ -164,7 +238,11 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" } diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_countries.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_countries.json new file mode 100644 index 00000000..0775d985 --- /dev/null +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_countries.json @@ -0,0 +1,250 @@ +{ + "type": "record", + "name": "top_server_countries", + "namespace": "druid", + "doc": { + "partition_key": "__time", + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "time": [ + "__time" + ], + "dimensions": [ + "server_country" + ], + "metrics": [ + "sessions", + "bytes", + "in_bytes", + "out_bytes", + "pkts", + "in_pkts", + "out_pkts" + ], + "filters": [ + "rank_by", + "device_group", + "data_center" + ], + "references": { + "$ref": "public_schema_info.json#/schema_query/references" + } + }, + "measurements": { + "ingestion_period": "PT1M", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "bytes": "bytes", + "in_bytes": "bytes", + "out_bytes": "bytes", + "pkts": "packets", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "bytes": "Bps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "pkts": "pps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "bytes": "bps", + "in_bytes": "bps", + "out_bytes": "bps" + } + } + } + }, + "fields": [ + { + "name": "__time", + "label": "Time", + "type": { + "type": "string", + "logicalType": "timestamp" + }, + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "enabled" + } + }, + { + "name": "vsys_id", + "label": "Vsys ID", + "doc": { + "visibility": "enabled" + }, + "type": "long", + "default": 1 + }, + { + "name": "device_id", + "label": "Device ID", + "doc": { + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "device_group", + "label": "Device Group", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "visibility": "enabled" + } + }, + { + "name": "data_center", + "label": "Data Center", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,in" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "visibility": "enabled" + } + }, + { + "name": "rank_by", + "label": "Rank", + "doc": { + "constraints": { + "operator_functions": "=" + }, + "data": [ + { + "code": "bytes", + "value": "Bytes" + }, + { + "code": "sessions", + "value": "Sessions" + }, + { + "code": "packets", + "value": "Packets" + } + ], + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "server_country", + "label": "Server Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled" + }, + "type": "string" + }, + { + "name": "sessions", + "label": "Sessions", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "bytes", + "label": "Bytes", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "in_bytes", + "label": "Incoming Bytes", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "out_bytes", + "label": "Outgoing Bytes", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "out_pkts", + "label": "Outgoing Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_domains.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_domains.json index 6ddee6be..904b8626 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_domains.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_domains.json @@ -8,44 +8,58 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "domain" + "server_domain" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "domain", "sessions", + "bytes", "in_bytes", "out_bytes", + "pkts", "in_pkts", "out_pkts" ], "filters": [ - "vsys_id", - "device_id", + "rank_by", "device_group", - "data_center", - "domain", - "sessions", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 300, - "ingestion_delay": 15 + "ingestion_period": "PT1M", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "bytes": "bytes", + "in_bytes": "bytes", + "out_bytes": "bytes", + "pkts": "packets", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "bytes": "Bps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "pkts": "pps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "bytes": "bps", + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -69,7 +83,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -113,17 +127,37 @@ } }, { - "name": "domain", - "label": "Domain", + "name": "rank_by", + "label": "Rank", "doc": { + "constraints": { + "operator_functions": "=" + }, + "data": [ + { + "code": "bytes", + "value": "Bytes" + }, + { + "code": "sessions", + "value": "Sessions" + }, + { + "code": "packets", + "value": "Packets" + } + ], "visibility": "enabled" }, "type": "string" }, { - "name": "metric", - "label": "Metric", + "name": "server_domain", + "label": "Server Domain", "doc": { + "constraints": { + "type": "domain" + }, "visibility": "enabled" }, "type": "string" @@ -132,23 +166,35 @@ "name": "sessions", "label": "Sessions", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, { - "name": "in_bytes", - "label": "Incoming Bytes", + "name": "bytes", + "label": "Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, { - "name": "in_pkts", - "label": "Incoming Packets", + "name": "in_bytes", + "label": "Incoming Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, @@ -156,7 +202,35 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, @@ -164,7 +238,11 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" } diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_fqdns.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_fqdns.json index fe2cd1bb..6690232e 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_fqdns.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_fqdns.json @@ -8,44 +8,58 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "fqdn" + "server_fqdn" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "fqdn", "sessions", + "bytes", "in_bytes", "out_bytes", + "pkts", "in_pkts", "out_pkts" ], "filters": [ - "vsys_id", - "device_id", + "rank_by", "device_group", - "data_center", - "fqdn", - "sessions", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 300, - "ingestion_delay": 15 + "ingestion_period": "PT1M", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "bytes": "bytes", + "in_bytes": "bytes", + "out_bytes": "bytes", + "pkts": "packets", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "bytes": "Bps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "pkts": "pps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "bytes": "bps", + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -69,7 +83,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -113,17 +127,37 @@ } }, { - "name": "fqdn", - "label": "FQDN", + "name": "rank_by", + "label": "Rank", "doc": { + "constraints": { + "operator_functions": "=" + }, + "data": [ + { + "code": "bytes", + "value": "Bytes" + }, + { + "code": "sessions", + "value": "Sessions" + }, + { + "code": "packets", + "value": "Packets" + } + ], "visibility": "enabled" }, "type": "string" }, { - "name": "metric", - "label": "Metric", + "name": "server_fqdn", + "label": "Server FQDN", "doc": { + "constraints": { + "type": "fqdn" + }, "visibility": "enabled" }, "type": "string" @@ -132,23 +166,35 @@ "name": "sessions", "label": "Sessions", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, { - "name": "in_bytes", - "label": "Incoming Bytes", + "name": "bytes", + "label": "Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, { - "name": "in_pkts", - "label": "Incoming Packets", + "name": "in_bytes", + "label": "Incoming Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, @@ -156,7 +202,35 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, @@ -164,7 +238,11 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" } diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_ips.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_ips.json index bf245f75..f0f55f4c 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_ips.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/top_server_ips.json @@ -8,44 +8,58 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "server_ip" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "server_ip", "sessions", + "bytes", "in_bytes", "out_bytes", + "pkts", "in_pkts", "out_pkts" ], "filters": [ - "vsys_id", - "device_id", + "rank_by", "device_group", - "data_center", - "server_ip", - "sessions", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts" + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 300, - "ingestion_delay": 15 + "ingestion_period": "PT1M", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "sessions": "sessions", + "bytes": "bytes", + "in_bytes": "bytes", + "out_bytes": "bytes", + "pkts": "packets", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "sessions": "sps", + "bytes": "Bps", + "in_bytes": "Bps", + "out_bytes": "Bps", + "pkts": "pps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "bytes": "bps", + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -69,7 +83,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -113,17 +127,37 @@ } }, { - "name": "server_ip", - "label": "Server IP", + "name": "rank_by", + "label": "Rank", "doc": { + "constraints": { + "operator_functions": "=" + }, + "data": [ + { + "code": "bytes", + "value": "Bytes" + }, + { + "code": "sessions", + "value": "Sessions" + }, + { + "code": "packets", + "value": "Packets" + } + ], "visibility": "enabled" }, "type": "string" }, { - "name": "metric", - "label": "Metric", + "name": "server_ip", + "label": "Server IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled" }, "type": "string" @@ -132,23 +166,35 @@ "name": "sessions", "label": "Sessions", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, { - "name": "in_bytes", - "label": "Incoming Bytes", + "name": "bytes", + "label": "Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, { - "name": "in_pkts", - "label": "Incoming Packets", + "name": "in_bytes", + "label": "Incoming Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } }, "type": "long" }, @@ -156,7 +202,35 @@ "name": "out_bytes", "label": "Outgoing Bytes", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + } + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } + }, + "type": "long" + }, + { + "name": "in_pkts", + "label": "Incoming Packets", + "doc": { + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" }, @@ -164,7 +238,11 @@ "name": "out_pkts", "label": "Outgoing Packets", "doc": { - "visibility": "enabled" + "visibility": "enabled", + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + } }, "type": "long" } diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_general_stat.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_general_stat.json index c6a31f2e..b9c2f5e3 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_general_stat.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_general_stat.json @@ -8,57 +8,62 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", "device_group", "data_center" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "sessions", - "active_sessions", "closed_sessions", "in_bytes", "out_bytes", "in_pkts", "out_pkts", "asymmetric_c2s_flows", - "asymmetric_s2c_flows", - "tcp_syn_pkts", - "tcp_ack_pkts", - "tcp_fin_pkts", - "tcp_rst_pkts" + "asymmetric_s2c_flows" ], "filters": [ - "vsys_id", - "device_id", "device_group", - "data_center", - "sessions", - "active_sessions", - "closed_sessions", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts", - "asymmetric_c2s_flows", - "asymmetric_s2c_flows", - "tcp_syn_pkts", - "tcp_ack_pkts", - "tcp_fin_pkts", - "tcp_rst_pkts" + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 5, - "ingestion_delay": 15 + "ingestion_period": "PT1S", + "compaction_period": "PT1M", + "skip_compaction_offset_from_latest": "P7D", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "in_bytes": "bytes", + "out_bytes": "bytes", + "asymmetric_c2s_flows": "flows", + "asymmetric_s2c_flows": "flows", + "sessions": "sessions", + "closed_sessions": "sessions", + "in_pkts": "packets", + "out_pkts": "packets" + }, + "RATE": { + "in_bytes": "Bps", + "out_bytes": "Bps", + "asymmetric_c2s_flows": "fps", + "asymmetric_s2c_flows": "fps", + "sessions": "sps", + "closed_sessions": "sps", + "in_pkts": "pps", + "out_pkts": "pps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -122,13 +127,17 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { "name": "sessions", - "label": "Sessions", + "label": "New Sessions", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -137,6 +146,9 @@ "name": "active_sessions", "label": "Active Sessions", "doc": { + "constraints": { + "metric_type": "gauge" + }, "visibility": "enabled" }, "type": "long" @@ -145,6 +157,10 @@ "name": "closed_sessions", "label": "Closed Sessions", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" }, "type": "long" @@ -154,6 +170,10 @@ "label": "Incoming Bytes", "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" } }, @@ -162,6 +182,10 @@ "label": "Incoming Packets", "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, @@ -170,6 +194,10 @@ "label": "Outgoing Bytes", "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" } }, @@ -178,22 +206,34 @@ "label": "Outgoing Packets", "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, { "name": "asymmetric_c2s_flows", - "label": "Asymmetric Flows (c2s)", + "label": "Asymmetric Flows(c2s)", "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, { "name": "asymmetric_s2c_flows", - "label": "Asymmetric Flows (s2c)", + "label": "Asymmetric Flows(s2c)", "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, @@ -202,22 +242,46 @@ "label": "TCP SYN Packets", "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, { - "name": "tcp_syn_ack_pkts", - "label": "TCP SYN/ACK Packets", + "name": "tcp_ack_pkts", + "label": "TCP ACK Packets", "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, { - "name": "tcp_ack_pkts", - "label": "TCP ACK Packets", + "name": "tcp_fin_pkts", + "label": "TCP FIN Packets", + "type": "long", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + } + }, + { + "name": "tcp_rst_pkts", + "label": "TCP RST Packets", "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } } diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_shaping_rule_hits.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_shaping_rule_hits.json index 640b1551..f29981dc 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_shaping_rule_hits.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_shaping_rule_hits.json @@ -8,60 +8,59 @@ "$ref": "public_schema_info.json#/functions" }, "schema_query": { + "time": [ + "__time" + ], "dimensions": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "rule_id", "profile_id", - "priority" + "device_group", + "data_center" ], "metrics": [ - "vsys_id", - "device_id", - "device_group", - "data_center", - "rule_id", - "profile_id", - "priority", "in_bytes", "out_bytes", "in_pkts", "out_pkts", - "in_drop_pkt", - "out_drop_pkts", - "in_max_latency_us", - "out_max_latency_us", - "in_queue_len", - "out_queue_len" + "in_drop_pkts", + "out_drop_pkts" ], "filters": [ - "vsys_id", - "device_id", - "device_group", - "data_center", "rule_id", "profile_id", "priority", - "in_bytes", - "out_bytes", - "in_pkts", - "out_pkts", - "in_drop_pkt", - "out_drop_pkts", - "in_max_latency_us", - "out_max_latency_us", - "in_queue_len", - "out_queue_len" + "device_group", + "data_center" ], "references": { "$ref": "public_schema_info.json#/schema_query/references" } }, "measurements": { - "granularity": 1, - "ingestion_delay": 5 + "ingestion_period": "PT1S", + "ingestion_delay": 5, + "aggregated_metric_unit": { + "SUM": { + "in_bytes": "bytes", + "out_bytes": "bytes", + "in_pkts": "packets", + "out_pkts": "packets", + "in_drop_pkts": "packets", + "out_drop_pkts": "packets" + }, + "RATE": { + "in_bytes": "Bps", + "out_bytes": "Bps", + "in_pkts": "pps", + "out_pkts": "pps", + "in_drop_pkts": "pps", + "out_drop_pkts": "pps" + }, + "BITRATE": { + "in_bytes": "bps", + "out_bytes": "bps" + } + } } }, "fields": [ @@ -85,7 +84,7 @@ "doc": { "visibility": "enabled" }, - "type": "int", + "type": "long", "default": 1 }, { @@ -131,7 +130,7 @@ { "name": "rule_id", "label": "Rule", - "type": "int", + "type": "long", "doc": { "constraints": { "operator_functions": "=,in" @@ -142,7 +141,7 @@ { "name": "profile_id", "label": "Profile ID", - "type": "int", + "type": "long", "doc": { "visibility": "enabled" } @@ -150,7 +149,7 @@ { "name": "priority", "label": "Priority", - "type": "int", + "type": "long", "doc": { "visibility": "enabled" } @@ -158,80 +157,116 @@ { "name": "in_bytes", "label": "Incoming Bytes", - "type": "int", + "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" } }, { "name": "in_pkts", "label": "Incoming Packets", - "type": "int", + "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, { "name": "out_bytes", "label": "Outgoing Bytes", - "type": "int", + "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE, BITRATE" + }, "visibility": "enabled" } }, { "name": "out_pkts", "label": "Outgoing Packets", - "type": "int", + "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, { "name": "in_drop_pkts", "label": "Incoming Drop Packets", - "type": "int", + "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, { "name": "out_drop_pkts", "label": "Outgoing Drop Packets", - "type": "int", + "type": "long", "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, "visibility": "enabled" } }, { "name": "in_max_latency_us", - "label": "Incoming Max Latency (us)", - "type": "int", + "label": "Incoming Max Latency(us)", + "type": "long", "doc": { + "constraints": { + "metric_type": "gauge" + }, "visibility": "enabled" } }, { "name": "out_max_latency_us", - "label": "Outgoing Max Latency (us)", - "type": "int", + "label": "Outgoing Max Latency(us)", + "type": "long", "doc": { + "constraints": { + "metric_type": "gauge" + }, "visibility": "enabled" } }, { "name": "in_queue_len", "label": "Incoming Queue Length", - "type": "int", + "type": "long", "doc": { + "constraints": { + "metric_type": "gauge" + }, "visibility": "enabled" } }, { "name": "out_queue_len", "label": "Outgoing Queue Length", - "type": "int", + "type": "long", "doc": { + "constraints": { + "metric_type": "gauge" + }, "visibility": "enabled" } } diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_sketch_metric.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_sketch_metric.json new file mode 100644 index 00000000..25be505d --- /dev/null +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_sketch_metric.json @@ -0,0 +1,619 @@ +{ + "type": "record", + "name": "traffic_sketch_metric", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "log_id", + "partition_key": "recv_time", + "index_key": [ + "vsys_id", + "direction", + "ip_protocol", + "app", + "client_ip", + "recv_time" + ], + "ttl": null, + "default_ttl": 2592000, + "functions": { + "$ref": "public_schema_info.json#/functions" + }, + "schema_query": { + "time": [ + "recv_time" + ], + "dimensions": [ + "vsys_id", + "device_id", + "device_group", + "data_center", + "direction", + "ip_protocol", + "client_ip", + "server_ip", + "internal_ip", + "external_ip", + "client_country", + "server_country", + "client_asn", + "server_asn", + "server_fqdn", + "server_domain", + "app", + "app_category", + "c2s_ttl", + "s2c_ttl", + "c2s_link_id", + "s2c_link_id" + ], + "metrics": [ + "vsys_id", + "device_id", + "device_group", + "data_center", + "direction", + "ip_protocol", + "client_ip", + "server_ip", + "internal_ip", + "external_ip", + "client_country", + "server_country", + "client_asn", + "server_asn", + "server_fqdn", + "server_domain", + "app", + "app_category", + "c2s_ttl", + "s2c_ttl", + "c2s_link_id", + "s2c_link_id", + "sessions", + "bytes", + "sent_bytes", + "received_bytes", + "pkts", + "sent_pkts", + "received_pkts", + "asymmetric_c2s_flows", + "asymmetric_s2c_flows", + "c2s_fragments", + "s2c_fragments", + "c2s_tcp_lost_bytes", + "s2c_tcp_lost_bytes", + "c2s_tcp_retransmitted_pkts", + "s2c_tcp_retransmitted_pkts" + ], + "filters": [ + "log_id", + "vsys_id", + "device_id", + "device_group", + "data_center", + "direction", + "ip_protocol", + "client_ip", + "server_ip", + "internal_ip", + "external_ip", + "client_country", + "server_country", + "client_asn", + "server_asn", + "server_fqdn", + "server_domain", + "app", + "app_category", + "c2s_ttl", + "s2c_ttl", + "c2s_link_id", + "s2c_link_id", + "sessions", + "bytes", + "sent_bytes", + "received_bytes", + "pkts", + "sent_pkts", + "received_pkts", + "asymmetric_c2s_flows", + "asymmetric_s2c_flows", + "c2s_fragments", + "s2c_fragments", + "c2s_tcp_lost_bytes", + "s2c_tcp_lost_bytes", + "c2s_tcp_retransmitted_pkts", + "s2c_tcp_retransmitted_pkts" + ] + }, + "measurements": { + "ingestion_period": "PT1M" + } + }, + "fields": [ + { + "name": "recv_time", + "label": "Receive Time", + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled", + "ttl": null + }, + "type": { + "type": "long", + "logicalType": "unix_timestamp" + } + }, + { + "name": "log_id", + "label": "Log ID", + "doc": { + "constraints": { + "aggregation_functions": "COUNT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "vsys_id", + "label": "Vsys ID", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "long", + "default": 1 + }, + { + "name": "device_id", + "label": "Device ID", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "device_group", + "label": "Device Group", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']", + "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "device_tag.json#", + "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']", + "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "direction", + "label": "Direction", + "doc": { + "data": { + "$ref": "public_schema_info.json#/fields/direction/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "ip_protocol", + "label": "IP Protocol", + "doc": { + "data": { + "$ref": "public_schema_info.json#/fields/ip_protocol/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_ip", + "label": "Client IP", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_ip", + "label": "Server IP", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_country", + "label": "Client Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_country", + "label": "Server Country", + "doc": { + "constraints": { + "type": "country" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "client_asn", + "label": "Client ASN", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "server_asn", + "label": "Server ASN", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "server_fqdn", + "label": "Server FQDN", + "doc": { + "constraints": { + "type": "fqdn" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "server_domain", + "label": "Server Domain", + "doc": { + "constraints": { + "type": "domain" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "app", + "label": "Application", + "doc": { + "constraints": { + "type": "app" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "app_category", + "label": "Application Category", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { + "name": "c2s_ttl", + "label": "Client-to-Server TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { + "name": "s2c_ttl", + "label": "Server-to-Client TTL", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { + "name": "c2s_link_id", + "label": "Client-to-Server Link ID", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { + "name": "s2c_link_id", + "label": "Server-to-Client Link ID", + "doc": { + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { + "name": "sessions", + "label": "Sessions", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "bytes", + "label": "Bytes", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE", + "type": "bytes" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "sent_bytes", + "label": "Bytes Sent", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE", + "type": "bytes" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "received_bytes", + "label": "Bytes Received", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE", + "type": "bytes" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "pkts", + "label": "Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE", + "type": "decimal" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "sent_pkts", + "label": "Packets Sent", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE", + "type": "decimal" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "received_pkts", + "label": "Packets Received", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE", + "type": "decimal" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "asymmetric_c2s_flows", + "label": "Client-to-Server Asymmetric Flows", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "asymmetric_s2c_flows", + "label": "Server-to-Client Asymmetric Flows", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "c2s_fragments", + "label": "Client-to-Server Fragments", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "s2c_fragments", + "label": "Server-to-Client Fragments", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "long" + }, + { + "name": "c2s_tcp_lost_bytes", + "label": "Client-to-Server Lost Bytes", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "s2c_tcp_lost_bytes", + "label": "Server-to-Client Lost Bytes", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "c2s_tcp_retransmitted_pkts", + "label": "Client-to-Server Retransmission Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + }, + { + "name": "s2c_tcp_retransmitted_pkts", + "label": "Server-to-Client Retransmission Packets", + "doc": { + "constraints": { + "metric_type": "counter", + "aggregation_functions": "SUM, RATE" + }, + "visibility": "enabled" + }, + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_top_destination_ip_metrics_log.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_top_destination_ip_metrics_log.json deleted file mode 100644 index 081fe25b..00000000 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/traffic_top_destination_ip_metrics_log.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "type": "record", - "name": "traffic_top_destination_ip_metrics_log", - "namespace": "druid", - "doc": { - "partition_key": "__time", - "functions": { - "$ref": "public_schema_info.json#/functions" - }, - "schema_query": { - "dimensions": [ - "vsys_id", - "destination_ip", - "attack_type" - ], - "metrics": [ - "vsys_id", - "destination_ip", - "attack_type", - "session_rate", - "packet_rate", - "bit_rate", - "partition_num" - ], - "filters": [ - "vsys_id", - "destination_ip", - "attack_type", - "session_rate", - "packet_rate", - "bit_rate", - "partition_num" - ], - "references": { - "$ref": "public_schema_info.json#/schema_query/references" - } - } - }, - "fields": [ - { - "name": "__time", - "label": "Time", - "type": { - "type": "string", - "logicalType": "timestamp" - }, - "doc": { - "constraints": { - "type": "timestamp" - }, - "visibility": "enabled" - } - }, - { - "name": "vsys_id", - "label": "Vsys ID", - "doc": { - "visibility": "enabled" - }, - "type": "int", - "default": 1 - }, - { - "name": "destination_ip", - "label": "Destination IP", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "attack_type", - "label": "Attack type", - "doc": { - "visibility": "enabled" - }, - "type": "string" - }, - { - "name": "session_rate", - "label": "Sessions/s", - "type": "long", - "doc": { - "constraints": { - "type": "sessions/sec" - }, - "visibility": "enabled" - } - }, - { - "name": "packet_rate", - "label": "Packets/s", - "type": "long", - "doc": { - "constraints": { - "type": "packets/sec" - }, - "visibility": "enabled" - } - }, - { - "name": "bit_rate", - "label": "Bits/s", - "type": "long", - "doc": { - "constraints": { - "type": "bits/sec" - }, - "visibility": "enabled" - } - }, - { - "name": "partition_num", - "label": "Partition Num", - "doc": { - "visibility": "enabled" - }, - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/transaction_record.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/transaction_record.json index 1a5cc601..0e86a470 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/transaction_record.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/transaction_record.json @@ -7,17 +7,23 @@ "partition_key": "recv_time", "ttl": null, "default_ttl": 2592000, - "index_key": { - "$ref": "public_schema_info.json#/transaction_record/index_key" - }, + "index_key": [ + "vsys_id", + "session_id", + "recv_time" + ], "functions": { "$ref": "public_schema_info.json#/functions" }, "schema_query": { - "dimensions": [ - "session_id", + "time": [ + "recv_time", "processing_time", "ingestion_time", + "insert_time" + ], + "dimensions": [ + "session_id", "address_type", "decoded_as", "vsys_id", @@ -58,6 +64,7 @@ "mail_attachment_name", "mail_attachment_name_charset", "mail_eml_file", + "mail_starttls_flag", "dns_message_id", "dns_qr", "dns_opcode", @@ -87,6 +94,7 @@ "sip_bye" ], "metrics": [ + "log_id", "address_type", "decoded_as", "vsys_id", @@ -131,6 +139,7 @@ "mail_attachment_name", "mail_attachment_name_charset", "mail_eml_file", + "mail_starttls_flag", "dns_message_id", "dns_qr", "dns_opcode", @@ -145,8 +154,6 @@ "dns_nscount", "dns_arcount", "dns_qname", - "dns_qtype", - "dns_qclass", "dns_cname", "dns_sub", "dns_rr", @@ -175,7 +182,6 @@ "decoded_as", "vsys_id", "client_ip", - "internal_ip", "client_port", "server_ip", "server_port", @@ -216,6 +222,7 @@ "mail_attachment_name", "mail_attachment_name_charset", "mail_eml_file", + "mail_starttls_flag", "dns_message_id", "dns_qr", "dns_opcode", @@ -230,8 +237,6 @@ "dns_nscount", "dns_arcount", "dns_qname", - "dns_qtype", - "dns_qclass", "dns_cname", "dns_sub", "dns_rr", @@ -259,19 +264,224 @@ }, "decoded_as": { "BASE": { - "$ref": "public_schema_info.json#/decoded_as/BASE" + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "ingestion_time", + "processing_time", + "insert_time", + "address_type", + "vsys_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "server_ip", + "server_port" + ] }, "DNS": { - "$ref": "public_schema_info.json#/decoded_as/DNS" + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "ingestion_time", + "processing_time", + "insert_time", + "address_type", + "vsys_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "dns_qr", + "dns_qname", + "dns_qtype", + "server_ip", + "server_port" + ] }, "HTTP": { - "$ref": "public_schema_info.json#/decoded_as/HTTP" + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "ingestion_time", + "processing_time", + "insert_time", + "address_type", + "vsys_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "http_url", + "server_ip", + "server_port" + ] }, "MAIL": { - "$ref": "public_schema_info.json#/decoded_as/MAIL" + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "ingestion_time", + "processing_time", + "insert_time", + "address_type", + "vsys_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "mail_starttls_flag" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "mail_from", + "mail_to", + "mail_subject", + "server_ip", + "server_port" + ] }, "SIP": { - "$ref": "public_schema_info.json#/decoded_as/SIP" + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "ingestion_time", + "processing_time", + "insert_time", + "address_type", + "vsys_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "server_ip", + "server_port" + ] } }, "default_columns": [ @@ -310,11 +520,9 @@ "name": "log_id", "label": "Log ID", "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], + "constraints": { + "aggregation_functions": "COUNT" + }, "visibility": "enabled", "ttl": null }, @@ -325,7 +533,7 @@ "label": "Decoded AS", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { @@ -370,11 +578,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "current_timestamp" - } - ], "visibility": "enabled", "ttl": null }, @@ -390,15 +593,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "ingestion_time" - }, - { - "function": "get_value", - "appendTo": "recv_time" - } - ], "visibility": "enabled", "ttl": null }, @@ -429,16 +623,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/address_type/data" + }, "visibility": "enabled", "ttl": null }, @@ -459,6 +646,7 @@ "label": "Client IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, "visibility": "enabled", @@ -470,6 +658,9 @@ "name": "client_port", "label": "Client Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -480,14 +671,9 @@ "label": "Server IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "server_asn" - } - ], "visibility": "enabled", "ttl": null }, @@ -497,6 +683,9 @@ "name": "server_port", "label": "Server Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -554,6 +743,9 @@ "name": "http_url", "label": "HTTP.URL", "doc": { + "constraints": { + "type": "url" + }, "visibility": "enabled", "ttl": null }, @@ -588,16 +780,16 @@ }, { "name": "http_request_content_length", - "label": "HTTP.Request Content Length", + "label": "HTTP.Request Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_request_content_type", - "label": "HTTP.Request Content Type", + "label": "HTTP.Request Content-Type", "doc": { "visibility": "enabled", "ttl": null @@ -606,12 +798,12 @@ }, { "name": "http_response_content_length", - "label": "HTTP.Response Content Length", + "label": "HTTP.Response Content-Length", "doc": { "visibility": "enabled", "ttl": null }, - "type": "string" + "type": "long" }, { "name": "http_response_content_type", @@ -626,11 +818,6 @@ "name": "http_request_body", "label": "HTTP.Request Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -644,11 +831,6 @@ "name": "http_response_body", "label": "HTTP.Response Body", "doc": { - "format": [ - { - "function": "combine" - } - ], "allow_query": "false", "constraints": { "type": "file" @@ -662,7 +844,7 @@ "name": "http_proxy_flag", "label": "HTTP.Proxy Flag", "doc": { - "visibility": "enabled", + "visibility": "hidden", "ttl": null }, "type": "int" @@ -696,7 +878,7 @@ }, { "name": "http_user_agent", - "label": "HTTP.User Agent", + "label": "HTTP.User-Agent", "doc": { "visibility": "enabled", "ttl": null @@ -705,7 +887,7 @@ }, { "name": "http_set_cookie", - "label": "HTTP.Set Cookie", + "label": "HTTP.Set-Cookie", "doc": { "visibility": "enabled", "ttl": null @@ -725,6 +907,10 @@ "name": "http_status_code", "label": "HTTP.Status Code", "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT", + "type": "http_code" + }, "visibility": "enabled", "ttl": null }, @@ -732,7 +918,7 @@ }, { "name": "http_response_latency_ms", - "label": "HTTP.Response Latency (ms)", + "label": "HTTP.Response Latency(ms)", "doc": { "constraints": { "type": "decimal" @@ -740,7 +926,7 @@ "visibility": "enabled", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "http_action_file_size", @@ -752,11 +938,11 @@ "visibility": "enabled", "ttl": null }, - "type": "int" + "type": "long" }, { "name": "http_session_duration_ms", - "label": "HTTP.Session Duration (ms)", + "label": "HTTP.Session Duration(ms)", "doc": { "constraints": { "type": "decimal" @@ -764,11 +950,11 @@ "visibility": "enabled", "ttl": null }, - "type": "long" + "type": "int" }, { "name": "mail_protocol_type", - "label": "Mail.Protocol Type", + "label": "MAIL.Protocol Type", "doc": { "visibility": "enabled", "ttl": null @@ -777,8 +963,11 @@ }, { "name": "mail_account", - "label": "Mail.Account", + "label": "MAIL.Account", "doc": { + "constraints": { + "type": "email" + }, "visibility": "enabled", "ttl": null }, @@ -786,7 +975,7 @@ }, { "name": "mail_from_cmd", - "label": "Mail.From CMD", + "label": "MAIL.From CMD", "doc": { "visibility": "enabled", "ttl": null @@ -795,7 +984,7 @@ }, { "name": "mail_to_cmd", - "label": "Mail.To CMD", + "label": "MAIL.To CMD", "doc": { "visibility": "enabled", "ttl": null @@ -804,7 +993,7 @@ }, { "name": "mail_from", - "label": "Mail.From", + "label": "MAIL.From", "doc": { "constraints": { "type": "email" @@ -816,7 +1005,7 @@ }, { "name": "mail_password", - "label": "Mail.Password", + "label": "MAIL.Password", "doc": { "visibility": "enabled", "ttl": null @@ -825,7 +1014,7 @@ }, { "name": "mail_to", - "label": "Mail.To", + "label": "MAIL.To", "doc": { "constraints": { "type": "email" @@ -837,7 +1026,7 @@ }, { "name": "mail_cc", - "label": "Mail.CC", + "label": "MAIL.CC", "doc": { "visibility": "enabled", "ttl": null @@ -846,7 +1035,7 @@ }, { "name": "mail_bcc", - "label": "Mail.BCC", + "label": "MAIL.BCC", "doc": { "visibility": "enabled", "ttl": null @@ -855,14 +1044,8 @@ }, { "name": "mail_subject", - "label": "Mail.Subject", + "label": "MAIL.Subject", "doc": { - "format": [ - { - "function": "decode_of_base64", - "param": "$.mail_subject_charset" - } - ], "visibility": "enabled", "ttl": null }, @@ -870,7 +1053,7 @@ }, { "name": "mail_subject_charset", - "label": "Mail.Subject Charset", + "label": "MAIL.Subject Charset", "doc": { "visibility": "enabled", "ttl": null @@ -879,14 +1062,8 @@ }, { "name": "mail_attachment_name", - "label": "Mail.Attachment", + "label": "MAIL.Attachment Name", "doc": { - "format": [ - { - "function": "decode_of_base64", - "param": "$.mail_attachment_name_charset" - } - ], "visibility": "enabled", "ttl": null }, @@ -894,7 +1071,7 @@ }, { "name": "mail_attachment_name_charset", - "label": "Mail.Attachment Charset", + "label": "MAIL.Attachment Name Charset", "doc": { "visibility": "enabled", "ttl": null @@ -903,13 +1080,8 @@ }, { "name": "mail_eml_file", - "label": "Mail.EML File", + "label": "MAIL.EML File", "doc": { - "format": [ - { - "function": "combine" - } - ], "constraints": { "type": "file" }, @@ -919,6 +1091,21 @@ "type": "string" }, { + "name": "mail_starttls_flag", + "label": "MAIL.STARTTLS Flag", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": { + "$ref": "public_schema_info.json#/fields/mail_starttls_flag/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "int" + }, + { "name": "dns_message_id", "label": "DNS.Message ID", "doc": { @@ -934,16 +1121,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "QUERY" - }, - { - "code": "1", - "value": "RESPONSE" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_qr/data" + }, "visibility": "enabled", "ttl": null }, @@ -956,24 +1136,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "QUERY" - }, - { - "code": "1", - "value": "IQUERY" - }, - { - "code": "2", - "value": "STATUS" - }, - { - "code": "5", - "value": "UPDATE" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_opcode/data" + }, "visibility": "enabled", "ttl": null }, @@ -1019,76 +1184,9 @@ "name": "dns_rcode", "label": "DNS.RCODE", "doc": { - "data": [ - { - "code": 0, - "value": "NoError" - }, - { - "code": 1, - "value": "FormErr" - }, - { - "code": 2, - "value": "ServFail" - }, - { - "code": 3, - "value": "NXDomain" - }, - { - "code": 4, - "value": "NotImp" - }, - { - "code": 5, - "value": "Refused" - }, - { - "code": 6, - "value": "YXDomain" - }, - { - "code": 7, - "value": "YXRRSet" - }, - { - "code": 8, - "value": "NXRRSet" - }, - { - "code": 9, - "value": "NotAuth" - }, - { - "code": 10, - "value": "NotZone" - }, - { - "code": 16, - "value": "BADSIG" - }, - { - "code": 17, - "value": "BADKEY" - }, - { - "code": 18, - "value": "BADTIME" - }, - { - "code": 19, - "value": "BADMODE" - }, - { - "code": 20, - "value": "BADNAME" - }, - { - "code": 21, - "value": "BADALG" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_rcode/data" + }, "visibility": "enabled", "ttl": null }, @@ -1156,330 +1254,12 @@ "label": "DNS.QTYPE", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": { + "$ref": "public_schema_info.json#/fields/dns_qtype/data" }, - "data": [ - { - "code": "1", - "value": "A" - }, - { - "code": "2", - "value": "NS" - }, - { - "code": "3", - "value": "MD" - }, - { - "code": "4", - "value": "MF" - }, - { - "code": "5", - "value": "CNAME" - }, - { - "code": "6", - "value": "SOA" - }, - { - "code": "7", - "value": "MB" - }, - { - "code": "8", - "value": "MG" - }, - { - "code": "9", - "value": "MR" - }, - { - "code": "10", - "value": "NULL" - }, - { - "code": "11", - "value": "WKS" - }, - { - "code": "12", - "value": "PTR" - }, - { - "code": "13", - "value": "HINFO" - }, - { - "code": "14", - "value": "MINFO" - }, - { - "code": "15", - "value": "MX" - }, - { - "code": "16", - "value": "TXT" - }, - { - "code": "17", - "value": "RP" - }, - { - "code": "18", - "value": "AFSDB" - }, - { - "code": "19", - "value": "X25" - }, - { - "code": "20", - "value": "ISDN" - }, - { - "code": "21", - "value": "RT" - }, - { - "code": "22", - "value": "NSAP" - }, - { - "code": "23", - "value": "NSAP" - }, - { - "code": "24", - "value": "SIG" - }, - { - "code": "25", - "value": "KEY" - }, - { - "code": "26", - "value": "PX" - }, - { - "code": "27", - "value": "GPOS" - }, - { - "code": "28", - "value": "AAAA" - }, - { - "code": "29", - "value": "LOC" - }, - { - "code": "30", - "value": "EID" - }, - { - "code": "31", - "value": "NIMLOC" - }, - { - "code": "32", - "value": "NB" - }, - { - "code": "33", - "value": "SRV" - }, - { - "code": "34", - "value": "ATMA" - }, - { - "code": "35", - "value": "NAPTR" - }, - { - "code": "36", - "value": "KX" - }, - { - "code": "37", - "value": "CERT" - }, - { - "code": "38", - "value": "A6" - }, - { - "code": "39", - "value": "DNAME" - }, - { - "code": "40", - "value": "SINK" - }, - { - "code": "41", - "value": "OPT" - }, - { - "code": "42", - "value": "APL" - }, - { - "code": "43", - "value": "DS" - }, - { - "code": "44", - "value": "SSHFP" - }, - { - "code": "45", - "value": "IPSECKEY" - }, - { - "code": "46", - "value": "RRSIG" - }, - { - "code": "47", - "value": "NSEC" - }, - { - "code": "48", - "value": "DNSKEY" - }, - { - "code": "49", - "value": "DHCID" - }, - { - "code": "50", - "value": "NSEC3" - }, - { - "code": "51", - "value": "NSEC3PARAM" - }, - { - "code": "52", - "value": "TLSA" - }, - { - "code": "53", - "value": "SMIMEA" - }, - { - "code": "55", - "value": "HIP" - }, - { - "code": "59", - "value": "CDS" - }, - { - "code": "60", - "value": "CDNSKEY" - }, - { - "code": "61", - "value": "OPENPGPKEY" - }, - { - "code": "62", - "value": "CSYNC" - }, - { - "code": "63", - "value": "ZONEMD" - }, - { - "code": "64", - "value": "SVCB" - }, - { - "code": "65", - "value": "HTTPS" - }, - { - "code": "99", - "value": "SPF" - }, - { - "code": "100", - "value": "UINFO" - }, - { - "code": "101", - "value": "UID" - }, - { - "code": "102", - "value": "GID" - }, - { - "code": "103", - "value": "UNSPEC" - }, - { - "code": "108", - "value": "EUI48" - }, - { - "code": "109", - "value": "EUI64" - }, - { - "code": "249", - "value": "TKEY" - }, - { - "code": "250", - "value": "TSIG" - }, - { - "code": "251", - "value": "IXFR" - }, - { - "code": "252", - "value": "AXFR" - }, - { - "code": "253", - "value": "MAILB" - }, - { - "code": "254", - "value": "MAILA" - }, - { - "code": "255", - "value": "*" - }, - { - "code": "256", - "value": "URI" - }, - { - "code": "257", - "value": "CAA" - }, - { - "code": "32768", - "value": "TA" - }, - { - "code": "32769", - "value": "DLV" - }, - { - "code": "65521", - "value": "INTEGRITY" - } - ], "visibility": "enabled", "ttl": null }, @@ -1510,16 +1290,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "1", - "value": "DNS" - }, - { - "code": "2", - "value": "DNSSEC" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/dns_sub/data" + }, "visibility": "enabled", "ttl": null }, @@ -1536,12 +1309,12 @@ }, { "name": "dns_response_latency_ms", - "label": "DNS.Response Latency (ms)", + "label": "DNS.Response Latency(ms)", "doc": { "constraints": { "type": "decimal" }, - "visibility": "disabled", + "visibility": "enabled", "ttl": null }, "type": "int" @@ -1595,6 +1368,9 @@ "name": "sip_originator_sdp_connect_ip", "label": "SIP.Originator IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -1604,6 +1380,9 @@ "name": "sip_originator_sdp_media_port", "label": "SIP.Originator Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -1631,6 +1410,9 @@ "name": "sip_responder_sdp_connect_ip", "label": "SIP.Responder IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -1640,6 +1422,9 @@ "name": "sip_responder_sdp_media_port", "label": "SIP.Responder Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -1665,7 +1450,7 @@ }, { "name": "sip_duration_s", - "label": "SIP.Duration (s)", + "label": "SIP.Duration(s)", "doc": { "constraints": { "type": "decimal" diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/version.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/version.json index ca36a907..b489494a 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/version.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/version.json @@ -1,95 +1,68 @@ { - "product": "185 Environment", - "version": "21.11", - "registered": "Geedge", - "updated": "2020-08-01 00:00:00", - "components": { - "oss": [ - { - "name": "zookeeper", - "version": "3.4.9", - "licenseType": "Apache License 2.0", - "description": "分布式应用程序协调服务" - }, - { - "name": "kafka", - "version": "2.11_1.0.0", - "licenseType": "Apache License 2.0", - "description": "消息队列" - }, - { - "name": "habse", - "version": "2.2.3", - "licenseType": "Apache License 2.0", - "description": "用于文件系统和存储Radius数据" - }, - { - "name": "flume", - "version": "1.9.0", - "licenseType": "Apache License 2.0", - "description": "日志补全传输" - }, - { - "name": "clickhouse", - "version": "19.17.4.11", - "licenseType": "Apache License 2.0", - "description": "原始日志数据库" - }, - { - "name": "druid", - "version": "0.16.0", - "licenseType": "Apache License 2.0", - "description": "分析实时数据并提供低延迟查询的OLAP应用程序" - }, - { - "name": "gohangout", - "version": "1.14.4.20200616", - "description": "动态获取原始日志表schema入库程序" - } - ], - "apps": [ - { - "name": "galaxy-qgw-service", - "version": "v3.2.200730", - "description": "数据平台对外统一查询网关" - }, - { - "name": "galaxy-report-service", - "version": "20.07.08", - "description": "自定义报表查询服务" - }, - { - "name": "galaxy-hos-service", - "version": "20.07.17", - "description": "对象存储服务" - }, - { - "name": "xxl-job-admin", - "version": "v1.0.2.20200506", - "description": "分布式任务调度平台" - }, - { - "name": "xxl-job", - "version": "v1.0.2.20200628", - "description": "分布式任务调度平台-执行器" - } - ], - "tasks": [ - { - "name": "flume", - "version": "flume-config-20.08", - "description": "原始日志补全、subscriber更新、Radius上下线功能" - }, - { - "name": "druid", - "version": "druid-config-20.08", - "description": "所有分析日志任务" - }, - { - "name": "gohangout", - "version": "gohangout-config-20.08", - "description": "原始日志入库、上下线日志入库" - } - ] + "product": "TSG OLAP", + "version": "24.01", + "registered": "Geedge Networks", + "updated": "2024-01-23 12:00:00", + "components": [ + { + "name": "zookeeper", + "version": "3.5.9", + "description": "分布式应用程序协调服务" + }, + { + "name": "kafka", + "version": "3.4.1", + "description": "消息队列" + }, + { + "name": "Hadoop", + "version": "2.7.1", + "description": "分布式文件系统用于HBase、Druid等组件深度存储" + }, + { + "name": "hbase", + "version": "2.2.3", + "description": "用于文件系统和存储Radius数据" + }, + { + "name": "Flink", + "version": "1.13.1", + "description": "流数据计算框架用于日志预处理及部分统计" + }, + { + "name": "clickHouse", + "version": "21.8.13.1", + "description": "OLAP Analytic storage" + }, + { + "name": "druid", + "version": "26.0.0", + "description": "分析实时数据并提供低延迟查询的OLAP应用程序" + }, + { + "name":"nacos", + "version": "2.0.2", + "description": "分布式配置中心" + }, + { + "name": "hos", + "version": "20.07.17", + "description": "对象存储服务" + }, + { + "name": "jobAdmin", + "version": "2.2.0", + "description": "分布式任务调度平台" + }, + { + "name": "mariaDB", + "version": "10.5.3", + "description": "传统数据库用于nacos/druid/galaxy-job-service/dataset数据存储" + }, + { + "name":"arango", + "version": "3.6.4", + "description": "图数据库用于存储IPlearning统计结果" } + ] }
\ No newline at end of file diff --git a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/voip_record.json b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/voip_record.json index a4437f71..0be6d8dc 100644 --- a/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/voip_record.json +++ b/config/nacos/config/fixed-127.0.0.1_8848-tsg_nacos/data/config-data-tenant/tsg/Galaxy/voip_record.json @@ -5,21 +5,29 @@ "doc": { "primary_key": "log_id", "partition_key": "recv_time", - "index_key": { - "$ref": "public_schema_info.json#/voip_record/index_key" - }, + "index_key": [ + "vsys_id", + "decoded_as", + "data_center", + "device_group", + "recv_time" + ], "ttl": null, "default_ttl": 2592000, "functions": { "$ref": "public_schema_info.json#/functions" }, "schema_query": { - "dimensions": [ - "session_id", + "time": [ + "recv_time", "start_timestamp_ms", "end_timestamp_ms", "processing_time", "ingestion_time", + "insert_time" + ], + "dimensions": [ + "session_id", "device_id", "out_link_id", "in_link_id", @@ -28,6 +36,7 @@ "device_group", "address_type", "decoded_as", + "direction", "vsys_id", "flags", "flags_identify_info", @@ -35,32 +44,21 @@ "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", - "subscriber_id", - "imei", - "imsi", - "apn", - "phone_number", - "packet_capture_file", - "in_src_mac", - "out_src_mac", "server_ip", "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", - "server_fqdn", - "server_domain", - "in_dest_mac", - "out_dest_mac", - "app_path", - "app", - "app_debug_info", - "app_content", - "protocol_path", - "dup_traffic_flag", - "tunnel_endpoint_a_desc", - "tunnel_endpoint_b_desc", + "ip_protocol", "sip_call_id", "sip_originator_description", "sip_responder_description", @@ -80,79 +78,34 @@ "rtp_originator_dir" ], "metrics": [ - "start_timestamp_ms", - "end_timestamp_ms", - "device_id", - "data_center", - "sled_ip", - "device_group", - "address_type", - "decoded_as", - "vsys_id", - "security_mirrored_pkts", - "security_mirrored_bytes", + "log_id", "client_ip", "client_port", - "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", - "subscriber_id", - "imei", - "imsi", - "apn", - "phone_number", - "packet_capture_file", - "in_src_mac", - "out_src_mac", "server_ip", "server_port", - "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", - "server_fqdn", - "server_domain", - "in_dest_mac", - "out_dest_mac", - "app_path", - "app_debug_info", - "app", - "app_content", - "protocol_path", "sent_pkts", "received_pkts", "sent_bytes", "received_bytes", "duration_ms", "tcp_handshake_latency_ms", - "tcp_c2s_ip_fragments", - "tcp_s2c_ip_fragments", - "tcp_c2s_lost_bytes", - "tcp_s2c_lost_bytes", - "tcp_c2s_o3_pkts", - "tcp_s2c_o3_pkts", - "tcp_c2s_rtx_pkts", - "tcp_s2c_rtx_pkts", - "tcp_c2s_rtx_bytes", - "tcp_s2c_rtx_bytes", - "tcp_rtt_ms", "sip_call_id", - "sip_originator_description", - "sip_responder_description", - "sip_user_agent", "sip_server", "sip_originator_sdp_connect_ip", - "sip_originator_sdp_media_port", - "sip_originator_sdp_media_type", - "sip_originator_sdp_content", "sip_responder_sdp_connect_ip", - "sip_responder_sdp_media_port", - "sip_responder_sdp_media_type", - "sip_responder_sdp_content", - "sip_duration_s", - "sip_bye", - "rtp_payload_type_c2s", - "rtp_payload_type_s2c", - "rtp_originator_dir" + "sip_duration_s" ], "filters": [ "session_id", @@ -163,61 +116,40 @@ "processing_time", "ingestion_time", "device_id", + "out_link_id", + "in_link_id", "data_center", "sled_ip", "device_group", "address_type", "decoded_as", + "direction", "vsys_id", "flags", "flags_identify_info", - "security_mirrored_pkts", - "security_mirrored_bytes", "client_ip", "client_port", "client_os_desc", "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", "client_asn", - "subscriber_id", - "imei", - "imsi", - "apn", - "phone_number", - "in_src_mac", - "out_src_mac", "server_ip", "server_port", "server_os_desc", "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", "server_asn", - "server_fqdn", - "server_domain", - "in_dest_mac", - "out_dest_mac", - "app_path", - "app_debug_info", - "app", - "app_content", - "dup_traffic_flag", - "tunnel_endpoint_a_desc", - "tunnel_endpoint_b_desc", - "protocol_path", - "fqdn_category_list", + "ip_protocol", "sent_pkts", "received_pkts", "sent_bytes", "received_bytes", - "tcp_c2s_ip_fragments", - "tcp_s2c_ip_fragments", - "tcp_c2s_lost_bytes", - "tcp_s2c_lost_bytes", - "tcp_c2s_o3_pkts", - "tcp_s2c_o3_pkts", - "tcp_c2s_rtx_pkts", - "tcp_s2c_rtx_pkts", - "tcp_c2s_rtx_bytes", - "tcp_s2c_rtx_bytes", - "tcp_rtt_ms", "sip_call_id", "sip_originator_description", "sip_responder_description", @@ -241,46 +173,398 @@ "$ref": "public_schema_info.json#/schema_query/references" }, "details": { - "$ref": "public_schema_info.json#/details" + "general": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "direction", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info" + ], + "source": [ + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", + "client_asn" + ], + "destination": [ + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", + "server_asn" + ], + "protocol": [ + "ip_protocol", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "transmission": [ + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes" + ] } }, "decoded_as": { - "BASE": { - "$ref": "public_schema_info.json#/decoded_as/BASE" - }, "SIP": { - "$ref": "public_schema_info.json#/decoded_as/SIP" + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "direction", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", + "client_asn", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", + "server_asn", + "ip_protocol", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "server_ip", + "server_port" + ] }, "RTP": { - "$ref": "public_schema_info.json#/decoded_as/RTP" + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "direction", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", + "client_asn", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", + "server_asn", + "ip_protocol", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "server_ip", + "server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + }, + "VoIP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "direction", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_country", + "client_super_administrative_area", + "client_administrative_area", + "client_sub_administrative_area", + "client_asn", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_country", + "server_super_administrative_area", + "server_administrative_area", + "server_sub_administrative_area", + "server_asn", + "ip_protocol", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "server_ip", + "server_port", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "rtp_pcap_path", + "rtp_originator_dir" + ] } }, "default_columns": [ "recv_time", - "subscriber_id", "client_ip", "client_port", "server_ip", "server_port", - "decoded_as", - "server_fqdn" + "decoded_as" ], "internal_columns": [ "recv_time", "log_id", - "flags_identify_info", - "tunnels", - "app_debug_info", - "app_content", - "packet_capture_file", - "tunnel_endpoint_a_desc", - "tunnel_endpoint_b_desc" + "flags_identify_info" ], "tunnel_type": { "$ref": "public_schema_info.json#/tunnel_type" }, "measurements": { - "aggregates": { + "aggregated_metric_unit": { + "SUM": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes" + }, + "RATE": { + "sent_pkts": "pps", + "received_pkts": "pps", + "sent_bytes": "Bps", + "received_bytes": "Bps" + }, + "BITRATE": { + "sent_bytes": "bps", + "received_bytes": "bps" + }, + "MAX": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "sip_duration_s": "s" + }, + "MIN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "sip_duration_s": "s" + }, + "AVG": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "sip_duration_s": "s" + }, + "MEDIAN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "sip_duration_s": "s" + }, + "P95_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "sip_duration_s": "s" + }, + "P99_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "sip_duration_s": "s" + } + }, + "field_discovery_metric": { "sessions": [ { "fn": "count", @@ -340,11 +624,9 @@ "name": "log_id", "label": "Log ID", "doc": { - "format": [ - { - "function": "snowflake_id" - } - ], + "constraints": { + "aggregation_functions": "COUNT" + }, "visibility": "enabled", "ttl": null }, @@ -355,20 +637,20 @@ "label": "Decoded AS", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": [ { - "code": "BASE", - "value": "BASE" - }, - { "code": "SIP", "value": "SIP" }, { "code": "RTP", "value": "RTP" + }, + { + "code": "VoIP", + "value": "VoIP" } ], "visibility": "enabled", @@ -393,11 +675,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -414,11 +691,6 @@ "operator_functions": "=,!=,>,<,>=,<=", "type": "datetime64" }, - "format": [ - { - "function": "from_unix_timestamp_ms" - } - ], "visibility": "enabled", "ttl": null }, @@ -429,10 +701,11 @@ }, { "name": "duration_ms", - "label": "Duration (ms)", + "label": "Duration(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -441,10 +714,11 @@ }, { "name": "tcp_handshake_latency_ms", - "label": "TCP Handshake Latency (ms)", + "label": "TCP Handshake Latency(ms)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -458,11 +732,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "current_timestamp" - } - ], "visibility": "enabled", "ttl": null }, @@ -478,15 +747,6 @@ "constraints": { "type": "unix_timestamp" }, - "format": [ - { - "function": "ingestion_time" - }, - { - "function": "get_value", - "appendTo": "recv_time" - } - ], "visibility": "enabled", "ttl": null }, @@ -542,18 +802,6 @@ "label": "Device Tag", "doc": { "visibility": "hidden", - "format": [ - { - "function": "flattenSpec", - "appendTo": "data_center", - "param": "$.tags[?(@.tag=='data_center')][0].value" - }, - { - "function": "flattenSpec", - "appendTo": "device_group", - "param": "$.tags[?(@.tag=='device_group')][0].value" - } - ], "ttl": null }, "type": "string" @@ -563,7 +811,7 @@ "label": "Data Center", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -580,7 +828,7 @@ "label": "Device Group", "doc": { "constraints": { - "operator_functions": "=,!=,in,not in" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, "data": { "$ref": "device_tag.json#", @@ -597,6 +845,7 @@ "label": "Sled IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, "visibility": "enabled", @@ -611,22 +860,30 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "4", - "value": "ipv4" - }, - { - "code": "6", - "value": "ipv6" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/address_type/data" + }, "visibility": "enabled", "ttl": null }, "type": "int" }, { + "name": "direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in,notEmpty,empty" + }, + "data": { + "$ref": "public_schema_info.json#/fields/direction/data" + }, + "visibility": "enabled", + "ttl": null + }, + "type": "string" + }, + { "name": "vsys_id", "label": "Vsys ID", "doc": { @@ -675,311 +932,13 @@ "type": "string" }, { - "name": "security_rule_list", - "label": "Security Rule List", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "long", - "logicalType": "array" - } - }, - { - "name": "security_action", - "label": "Security Action", - "doc": { - "visibility": "hidden", - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "Deny", - "value": "Deny" - }, - { - "code": "Allow", - "value": "Allow" - } - ], - "ttl": null - }, - "type": "string" - }, - { - "name": "monitor_rule_list", - "label": "Monitor Rule List", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "long", - "logicalType": "array" - } - }, - { - "name": "sc_rule_list", - "label": "Service Chaining Rule List", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "long", - "logicalType": "array" - } - }, - { - "name": "sc_rsp_raw", - "label": "Service Chaining Rendered Service Path (Raw)", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "long", - "logicalType": "array" - } - }, - { - "name": "sc_rsp_decrypted", - "label": "Service Chaining Rendered Service Path (Decrypted)", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "long", - "logicalType": "array" - } - }, - { - "name": "shaping_rule_list", - "label": "Shaping Rule List", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "long", - "logicalType": "array" - } - }, - { - "name": "proxy_rule_list", - "label": "Proxy Rule List", - "doc": { - "constraints": { - "operator_functions": "has,notEmpty,empty" - }, - "visibility": "hidden", - "ttl": null - }, - "type": { - "type": "array", - "items": "long", - "logicalType": "array" - } - }, - { - "name": "proxy_action", - "label": "Proxy Action", - "doc": { - "visibility": "hidden", - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": [ - { - "code": "Intercept", - "value": "Intercept" - }, - { - "code": "No Intercept", - "value": "No Intercept" - } - ], - "ttl": null - }, - "type": "string" - }, - { - "name": "proxy_pinning_status", - "label": "Proxy Pinning Status", - "doc": { - "data": [ - { - "code": "0", - "value": "not pinning" - }, - { - "code": "1", - "value": "pinning" - }, - { - "code": "2", - "value": "maybe pinning" - } - ], - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "proxy_intercept_status", - "label": "Proxy Intercept Status", - "doc": { - "data": [ - { - "code": "0", - "value": "passthrough" - }, - { - "code": "1", - "value": "intercept" - }, - { - "code": "2", - "value": "shutdown" - } - ], - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "proxy_passthrough_reason", - "label": "Proxy Passthrough Reason", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "proxy_server_side_latency_ms", - "label": "Proxy Server Side Latency", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "proxy_client_side_latency_ms", - "label": "Proxy Client Side Latency", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "proxy_client_side_version", - "label": "Proxy Client Side Version", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "proxy_server_side_version", - "label": "Proxy Server Side Version", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "proxy_cert_verify", - "label": "Proxy Cert Verify", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "int" - }, - { - "name": "proxy_intercept_error", - "label": "Proxy Intercept Error", - "doc": { - "visibility": "hidden", - "ttl": null - }, - "type": "string" - }, - { - "name": "security_mirrored_pkts", - "label": "Security Mirrored Packets", - "type": "int", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - } - }, - { - "name": "security_mirrored_bytes", - "label": "Security Mirrored Bytes", - "type": "int", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - } - }, - { "name": "client_ip", "label": "Client IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "client_asn" - }, - { - "function": "radius_match", - "appendTo": "subscriber_id" - } - ], "visibility": "enabled", "ttl": null }, @@ -990,6 +949,10 @@ "label": "Client Port", "doc": { "visibility": "enabled", + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -1005,7 +968,7 @@ }, { "name": "client_geolocation", - "label": "Client Geolocation", + "label": "Client Geolocation(Deprecated)", "doc": { "visibility": "enabled", "ttl": null @@ -1013,44 +976,44 @@ "type": "string" }, { - "name": "client_asn", - "label": "Client ASN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "subscriber_id", - "label": "Subscriber ID", + "name": "client_country", + "label": "Client Country", "doc": { + "constraints": { + "type": "country" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "imei", - "label": "IMEI", + "name": "client_super_administrative_area", + "label": "Client Super Administrative Area", "doc": { + "constraints": { + "type": "super_administrative_area" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "imsi", - "label": "IMSI", + "name": "client_administrative_area", + "label": "Client Administrative Area", "doc": { + "constraints": { + "type": "administrative_area" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "apn", - "label": "APN", + "name": "client_sub_administrative_area", + "label": "Client SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1058,27 +1021,25 @@ "type": "string" }, { - "name": "phone_number", - "label": "Phone Number", + "name": "client_asn", + "label": "Client ASN", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, - "type": "string" + "type": "long" }, { "name": "server_ip", "label": "Server IP", "doc": { "constraints": { + "operator_functions": "=,!=,in,not in,like,not like,notEmpty,empty,isIPAddressInRange", "type": "ip" }, - "format": [ - { - "function": "geo_asn", - "appendTo": "server_asn" - } - ], "visibility": "enabled", "ttl": null }, @@ -1089,6 +1050,10 @@ "label": "Server Port", "doc": { "visibility": "enabled", + "constraints": { + "type": "port", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, "type": "int" @@ -1104,25 +1069,7 @@ }, { "name": "server_geolocation", - "label": "Server Geolocation", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "server_asn", - "label": "Server ASN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "server_fqdn", - "label": "Server FQDN", + "label": "Server Geolocation(Deprecated)", "doc": { "visibility": "enabled", "ttl": null @@ -1130,44 +1077,44 @@ "type": "string" }, { - "name": "server_domain", - "label": "Server Domain", + "name": "server_country", + "label": "Server Country", "doc": { + "constraints": { + "type": "country" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "app_path", - "label": "Application Path", - "type": "string", - "doc": { - "visibility": "enabled", - "ttl": null - } - }, - { - "name": "app", - "label": "Application", + "name": "server_super_administrative_area", + "label": "Server Super Administrative Area", "doc": { + "constraints": { + "type": "super_administrative_area" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "app_debug_info", - "label": "Application Debug Info", + "name": "server_administrative_area", + "label": "Server Administrative Area", "doc": { + "constraints": { + "type": "administrative_area" + }, "visibility": "enabled", "ttl": null }, "type": "string" }, { - "name": "app_content", - "label": "Application Content", + "name": "server_sub_administrative_area", + "label": "Server SubAdministrative Area", "doc": { "visibility": "enabled", "ttl": null @@ -1175,34 +1122,31 @@ "type": "string" }, { - "name": "protocol_path", - "label": "Protocol Path", + "name": "server_asn", + "label": "Server ASN", "doc": { "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, "ttl": null }, - "type": "string" + "type": "long" }, { - "name": "fqdn_category_list", - "label": "FQDN Category List", + "name": "ip_protocol", + "label": "IP Protocol", "doc": { "constraints": { - "operator_functions": "has,notEmpty,empty" + "operator_functions": "=,!=,in,not in,notEmpty,empty" }, - "dict_location": { - "path": "/v1/category/dict", - "key": "categoryId", - "value": "categoryName" + "data": { + "$ref": "public_schema_info.json#/fields/ip_protocol/data" }, "visibility": "enabled", "ttl": null }, - "type": { - "type": "array", - "items": "int", - "logicalType": "array" - } + "type": "string" }, { "name": "sent_pkts", @@ -1253,254 +1197,6 @@ "type": "long" }, { - "name": "tcp_c2s_ip_fragments", - "label": "TCP C2S IP Fragments ", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_s2c_ip_fragments", - "label": "TCP S2C IP Fragments", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_c2s_lost_bytes", - "label": "TCP C2S Lost Bytes", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_s2c_lost_bytes", - "label": "TCP S2C Lost Bytes", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_c2s_o3_pkts", - "label": "TCP C2S out-of-order packets", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_s2c_o3_pkts", - "label": "TCP S2C out-of-order packets", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_c2s_rtx_pkts", - "label": "TCP C2S Retransmission Packets", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_s2c_rtx_pkts", - "label": "TCP S2C Retransmission Packets", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_c2s_rtx_bytes", - "label": "TCP C2S Retransmission Bytes", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_s2c_rtx_bytes", - "label": "TCP S2C Retransmission Bytes", - "doc": { - "constraints": { - "type": "bytes" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_rtt_ms", - "label": "Round-trip Time", - "doc": { - "constraints": { - "type": "decimal" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "tcp_client_isn", - "label": "TCP Client ISN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "tcp_server_isn", - "label": "TCP Server ISN", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "long" - }, - { - "name": "packet_capture_file", - "label": "Packet Capture File", - "doc": { - "allow_query": "false", - "visibility": "enabled", - "constraints": { - "type": "file" - }, - "ttl": null - }, - "type": "string" - }, - { - "name": "in_src_mac", - "label": "Incoming Source MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "out_src_mac", - "label": "Outgoing Source MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "in_dest_mac", - "label": "Incoming Destination MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "out_dest_mac", - "label": "Outgoing Destination MAC", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "tunnels", - "label": "Tunnels", - "doc": { - "visibility": "enabled", - "ttl": null, - "format": [ - { - "function": "gtpc_match", - "appendTo": "imsi,imei,phone_number", - "param": "$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_a2b_teid,$.[?(@.tunnels_schema_type=='GTP')][0].gtp_endpoint_b2a_teid" - } - ] - }, - "type": "string" - }, - { - "name": "dup_traffic_flag", - "label": "Duplicate Traffic Flag", - "doc": { - "constraints": { - "operator_functions": "=,!=,in,not in" - }, - "data": { - "$ref": "public_schema_info.json#/fields/dup_traffic_flag/data" - }, - "visibility": "enabled", - "ttl": null - }, - "type": "int" - }, - { - "name": "tunnel_endpoint_a_desc", - "label": "Tunnel Endpoint A Description", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { - "name": "tunnel_endpoint_b_desc", - "label": "Tunnel Endpoint B Description", - "doc": { - "visibility": "enabled", - "ttl": null - }, - "type": "string" - }, - { "name": "sip_call_id", "label": "SIP.Call-ID", "doc": { @@ -1549,6 +1245,9 @@ "name": "sip_originator_sdp_connect_ip", "label": "SIP.Originator IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -1558,6 +1257,9 @@ "name": "sip_originator_sdp_media_port", "label": "SIP.Originator Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -1585,6 +1287,9 @@ "name": "sip_responder_sdp_connect_ip", "label": "SIP.Responder IP", "doc": { + "constraints": { + "type": "ip" + }, "visibility": "enabled", "ttl": null }, @@ -1594,6 +1299,9 @@ "name": "sip_responder_sdp_media_port", "label": "SIP.Responder Port", "doc": { + "constraints": { + "type": "port" + }, "visibility": "enabled", "ttl": null }, @@ -1619,10 +1327,11 @@ }, { "name": "sip_duration_s", - "label": "SIP.Duration (s)", + "label": "SIP.Duration(s)", "doc": { "constraints": { - "type": "decimal" + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" }, "visibility": "enabled", "ttl": null @@ -1640,121 +1349,14 @@ }, { "name": "rtp_payload_type_c2s", - "label": "RTP.Payload Type (c2s)", + "label": "RTP.Payload Type(Client-to-Server)", "doc": { "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "PCMU" - }, - { - "code": "1", - "value": "1016" - }, - { - "code": "2", - "value": "G721" - }, - { - "code": "3", - "value": "GSM" - }, - { - "code": "4", - "value": "G723" - }, - { - "code": "5", - "value": "DVI4_8000" - }, - { - "code": "6", - "value": "DVI4_16000" - }, - { - "code": "7", - "value": "LPC" - }, - { - "code": "8", - "value": "PCMA" - }, - { - "code": "9", - "value": "G722" - }, - { - "code": "10", - "value": "L16_STEREO" - }, - { - "code": "11", - "value": "L16_MONO" - }, - { - "code": "12", - "value": "QCELP" - }, - { - "code": "13", - "value": "CN" - }, - { - "code": "14", - "value": "MPA" - }, - { - "code": "15", - "value": "G728" - }, - { - "code": "16", - "value": "DVI4_11025" - }, - { - "code": "17", - "value": "DVI4_22050" - }, - { - "code": "18", - "value": "G729" - }, - { - "code": "19", - "value": "CN_OLD" - }, - { - "code": "25", - "value": "CELB" - }, - { - "code": "26", - "value": "JPEG" - }, - { - "code": "28", - "value": "NV" - }, - { - "code": "31", - "value": "H261" - }, - { - "code": "32", - "value": "MPV" - }, - { - "code": "33", - "value": "MP2T" - }, - { - "code": "34", - "value": "H263" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_payload_type_c2s/data" + }, "visibility": "enabled", "ttl": null }, @@ -1762,121 +1364,14 @@ }, { "name": "rtp_payload_type_s2c", - "label": "RTP.Payload Type (s2c)", + "label": "RTP.Payload Type(Server-to-Client)", "doc": { "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "PCMU" - }, - { - "code": "1", - "value": "1016" - }, - { - "code": "2", - "value": "G721" - }, - { - "code": "3", - "value": "GSM" - }, - { - "code": "4", - "value": "G723" - }, - { - "code": "5", - "value": "DVI4_8000" - }, - { - "code": "6", - "value": "DVI4_16000" - }, - { - "code": "7", - "value": "LPC" - }, - { - "code": "8", - "value": "PCMA" - }, - { - "code": "9", - "value": "G722" - }, - { - "code": "10", - "value": "L16_STEREO" - }, - { - "code": "11", - "value": "L16_MONO" - }, - { - "code": "12", - "value": "QCELP" - }, - { - "code": "13", - "value": "CN" - }, - { - "code": "14", - "value": "MPA" - }, - { - "code": "15", - "value": "G728" - }, - { - "code": "16", - "value": "DVI4_11025" - }, - { - "code": "17", - "value": "DVI4_22050" - }, - { - "code": "18", - "value": "G729" - }, - { - "code": "19", - "value": "CN_OLD" - }, - { - "code": "25", - "value": "CELB" - }, - { - "code": "26", - "value": "JPEG" - }, - { - "code": "28", - "value": "NV" - }, - { - "code": "31", - "value": "H261" - }, - { - "code": "32", - "value": "MPV" - }, - { - "code": "33", - "value": "MP2T" - }, - { - "code": "34", - "value": "H263" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_payload_type_s2c/data" + }, "visibility": "enabled", "ttl": null }, @@ -1902,20 +1397,9 @@ "constraints": { "operator_functions": "=,!=,in,not in" }, - "data": [ - { - "code": "0", - "value": "unknown" - }, - { - "code": "1", - "value": "c2s" - }, - { - "code": "2", - "value": "s2c" - } - ], + "data": { + "$ref": "public_schema_info.json#/fields/rtp_originator_dir/data" + }, "visibility": "enabled", "ttl": null }, diff --git a/dat/example_data.json b/dat/example_data.json new file mode 100644 index 00000000..3adc17fc --- /dev/null +++ b/dat/example_data.json @@ -0,0 +1,236 @@ +{ + "port": { + "mode": "range", + "start": 0, + "end": 65535 + }, + "http_code": { + "mode": "range", + "start": 100, + "end": 599 + }, + "country": { + "mode": "sequence", + "values": [ + "China", + "United States", + "Russia", + "American Samoa", + "Nauru", + "Italy", + "South Africa", + "Afghanistan", + "Germany", + "Canada", + "Myanmar", + "Kazakhsta", + "Pakistan", + "Japan", + "France" + ] + }, + "super_administrative_area": { + "mode": "sequence", + "values": [ + "Beijing", + "New York", + "Moscow Oblast", + "Eastern District", + "Meneng District", + "Piedmont", + "Gauteng", + "Nangarhar", + "Baden-Wurttemberg", + "Ontario", + "Yangon", + "Qostanay Oblysy", + "Punjab", + "Tokyo", + "Nouvelle-Aquitaine" + ] + }, + "administrative_area": { + "mode": "sequence", + "random": true, + "values": [ + "Beijing", + "Blossvale", + "Vlasikha", + "Utulei", + "Menen", + "Bruno", + "Irene", + "Jalalabad", + "Keltern", + "Southampton", + "San Chaung", + "Tenizovskiy", + "Chiniot", + "Sendagaya", + "Balzac" + ] + }, + "fqdn": { + "mode": "enum", + "values": [ + "appstore.ldmnq.com", + "oss.work.weixin.qq.com", + "extshort.weixin.qq.com", + "cube.weixinbridge.com", + "szextshort.weixin.qq.com", + "ctldl.windowsupdate.com", + "fsvippc.hw.kugou.com", + "i.work.weixin.qq.com", + "res.wx.qq.com", + "self.events.data.microsoft.com", + "docs.geedge.net", + "www.msftconnecttest.com", + "pds-app-cn-beijing.cn-beijing.log.aliyuncs.com", + "dns.google", + "mmbiz.qpic.cn", + "osfsr.lenovomm.com", + "static-pcs-sdk-server.alibaba.com", + "pypi.org", + "mobile.events.data.microsoft.com", + "tpstelemetry.tencent.com" + ] + }, + "domain": { + "mode": "enum", + "values": [ + "ldmnq.com", + "qq.com", + "microsoft.com", + "weixinbridge.com", + "baidu.com", + "kugou.com", + "windowsupdate.com", + "geedge.net", + "msftconnecttest.com", + "qpic.cn", + "google.com", + "aliyuncs.com", + "apple.com", + "alibaba.com", + "lenovomm.com", + "dns.google", + "360.cn", + "csdn.net", + "wps.cn", + "tencent.com" + ] + }, + "app": { + "mode": "enum", + "values": [ + "microsoft", + "iperf", + "baidu", + "iqiyi", + "qq_web", + "quic", + "cloudflare", + "wps_office", + "alibaba", + "360cn", + "tencent", + "apple", + "bing", + "amazon", + "bytedance", + "icloud", + "kingsoft", + "windows_marketplace", + "alibaba_cloud", + "qqmusic" + ] + }, + "ip": { + "mode": "enum", + "values": [ + "101.132.162.123", + "47.100.58.110", + "47.103.62.89", + "120.53.53.53", + "10.227.242.38", + "116.211.207.100", + "10.0.0.178", + "36.110.220.113", + "220.181.38.149", + "42.81.186.225", + "47.93.59.84", + "117.89.176.31", + "101.198.198.198", + "183.47.118.249", + "182.254.116.117", + "42.81.237.156", + "120.133.65.232", + "121.14.154.93", + "172.16.1.14", + "183.47.103.43", + "69.139.238.46", + "51.129.105.77", + "92.60.42.197", + "44.247.115.202", + "29.245.89.106" + ] + }, + "email": { + "mode": "enum", + "values": [ + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + "[email protected]", + ] + }, + "url": { + "mode": "enum", + "values": [ + "appstore.ldmnq.com/menu_manager", + "pds-app-cn-beijing.cn-beijing.log.aliyuncs.com/logstores/sync-app", + "www.msftconnecttest.com/connecttest.txt", + "captive.apple.com", + "static-pcs-sdk-server.alibaba.com/a", + "ipv6.msftconnecttest.com/connecttest.txt", + "d?dn=5736b7bfb4a72e8e87b41bbd2863bc1e2016d03acdea7eca8a31de0200a11452&id=2&ttl=1&type=aaaa", + "www.msftncsi.com/ncsi.txt", + "shuc-pc-hunt.ksord.com/", + "helpdubaclient.ksmobile.com/nep/v1/", + "go.microsoft.com/fwlink/?LinkID=252669&clcid=0x409", + "global.quickconnect.cn/Serv.php", + "conn-service-cn-04.allawntech.com/generate204", + "err.taobao.com/error1.html", + "qd-update.qq.com:8080/", + "software-down.keniu.com/config/k52zip/evluatesetcfg.dat", + "connectivitycheck-backup.platform.hihonorcloud.com/generate_204", + "weather2db.cmcm.com/ip/cityid", + "conf-darwin.xycdn.com/psdk_param?version=wymusicw_3.0.23.1", + "sdkoptedge.chinanetcenter.com/" + ] + }, + "hdr_histogram": { + "mode": "enum", + "values": [ + "[{\"value\":1,\"count\":1,\"percentile\":0.0016328130102540657},{\"value\":22,\"count\":44118,\"percentile\":72.03807719939913},{\"value\":23,\"count\":14684,\"percentile\":96.01430344196983},{\"value\":24,\"count\":1603,\"percentile\":98.63170269740709},{\"value\":25,\"count\":383,\"percentile\":99.2570700803344},{\"value\":26,\"count\":98,\"percentile\":99.4170857553393},{\"value\":28,\"count\":128,\"percentile\":99.62608582065182},{\"value\":30,\"count\":72,\"percentile\":99.74364835739011},{\"value\":31,\"count\":48,\"percentile\":99.8220233818823},{\"value\":33,\"count\":42,\"percentile\":99.89060152831298},{\"value\":35,\"count\":29,\"percentile\":99.93795310561035},{\"value\":37,\"count\":25,\"percentile\":99.9787734308667},{\"value\":39,\"count\":11,\"percentile\":99.99673437397949},{\"value\":41,\"count\":2,\"percentile\":100.0}]", + "[{\"value\":3,\"count\":1464,\"percentile\":7.778958554729011},{\"value\":8,\"count\":3478,\"percentile\":26.259298618490966},{\"value\":41,\"count\":4656,\"percentile\":50.99893730074389},{\"value\":49,\"count\":2828,\"percentile\":66.02550478214665},{\"value\":55,\"count\":2066,\"percentile\":77.00318809776833},{\"value\":59,\"count\":860,\"percentile\":81.57279489904357},{\"value\":119,\"count\":1118,\"percentile\":87.51328374070138},{\"value\":215,\"count\":698,\"percentile\":91.2221041445271},{\"value\":271,\"count\":860,\"percentile\":95.79171094580234},{\"value\":303,\"count\":344,\"percentile\":97.61955366631243},{\"value\":319,\"count\":158,\"percentile\":98.45908607863974},{\"value\":1087,\"count\":108,\"percentile\":99.03294367693942},{\"value\":1599,\"count\":38,\"percentile\":99.23485653560043},{\"value\":2943,\"count\":72,\"percentile\":99.61742826780021},{\"value\":3199,\"count\":20,\"percentile\":99.72369819341127},{\"value\":4351,\"count\":34,\"percentile\":99.90435706695006},{\"value\":7679,\"count\":10,\"percentile\":99.95749202975558},{\"value\":8703,\"count\":8,\"percentile\":100.0}]", + "[{\"value\":6,\"count\":1464,\"percentile\":7.752594789239568},{\"value\":8,\"count\":3498,\"percentile\":26.27621266680788},{\"value\":41,\"count\":4666,\"percentile\":50.984960813387},{\"value\":49,\"count\":2844,\"percentile\":66.04532937936878},{\"value\":55,\"count\":2074,\"percentile\":77.02817199745816},{\"value\":59,\"count\":860,\"percentile\":81.58229188731201},{\"value\":119,\"count\":1118,\"percentile\":87.502647744122},{\"value\":215,\"count\":698,\"percentile\":91.19889853844525},{\"value\":271,\"count\":870,\"percentile\":95.80597331073925},{\"value\":303,\"count\":344,\"percentile\":97.62762126668079},{\"value\":319,\"count\":158,\"percentile\":98.46430840923533},{\"value\":1087,\"count\":108,\"percentile\":99.03622113958907},{\"value\":1599,\"count\":38,\"percentile\":99.23744969286169},{\"value\":2943,\"count\":72,\"percentile\":99.61872484643084},{\"value\":3199,\"count\":20,\"percentile\":99.72463461131116},{\"value\":4351,\"count\":34,\"percentile\":99.90468121160771},{\"value\":7679,\"count\":10,\"percentile\":99.95763609404787},{\"value\":8703,\"count\":8,\"percentile\":100.0}]" + ] + } +} diff --git a/develop/checkstyle.xml b/develop/checkstyle.xml new file mode 100644 index 00000000..c1b4f7d2 --- /dev/null +++ b/develop/checkstyle.xml @@ -0,0 +1,444 @@ +<?xml version="1.0"?> +<!DOCTYPE module PUBLIC + "-//Puppy Crawl//DTD Check Configuration 1.3//EN" + "http://www.puppycrawl.com/dtds/configuration_1_3.dtd"> + +<!-- +This is a checkstyle configuration file. For descriptions of +what the following rules do, please see the checkstyle configuration +page at http://checkstyle.sourceforge.net/config.html. +--> + +<module name="Checker"> + + <module name="NewlineAtEndOfFile"> + <property name="fileExtensions" value="sh"/> + <!-- windows can use \r\n vs \n, so enforce the most used one ie UNIx style --> + <property name="lineSeparator" value="lf"/> + </module> + + <module name="RegexpSingleline"> + <!-- Checks that TODOs don't have stuff in parenthesis, e.g., username. --> + <property name="format" value="((//.*)|(\*.*))TODO\("/> + <property name="message" value="TODO comments must not include usernames."/> + <property name="severity" value="error"/> + </module> + + <module name="RegexpSingleline"> + <property name="format" value="\s+$"/> + <property name="message" value="Trailing whitespace"/> + <property name="severity" value="error"/> + </module> + + <module name="RegexpSingleline"> + <property name="format" value="Throwables.propagate\("/> + <property name="message" value="Throwables.propagate is deprecated"/> + <property name="severity" value="error"/> + </module> + + <!-- Prevent *Tests.java as tools may not pick them up --> + <module name="RegexpOnFilename"> + <property name="fileNamePattern" value=".*Tests\.java$"/> + </module> + + <module name="SuppressionFilter"> + <property name="file" value="${checkstyle.suppressions.file}" default="suppressions.xml"/> + </module> + + <module name="FileLength"> + <property name="max" value="3000"/> + </module> + + <!-- All Java AST specific tests live under TreeWalker module. --> + <module name="TreeWalker"> + + <!-- Allow use of comment to suppress javadocstyle --> + <module name="SuppressionCommentFilter"> + <property name="offCommentFormat" value="CHECKSTYLE.OFF\: ([\w\|]+)"/> + <property name="onCommentFormat" value="CHECKSTYLE.ON\: ([\w\|]+)"/> + <property name="checkFormat" value="$1"/> + </module> + + <!-- Prohibit T.getT() methods for standard boxed types --> + <module name="Regexp"> + <property name="format" value="Boolean\.getBoolean"/> + <property name="illegalPattern" value="true"/> + <property name="message" value="Use System.getProperties() to get system properties."/> + </module> + + <module name="Regexp"> + <property name="format" value="Integer\.getInteger"/> + <property name="illegalPattern" value="true"/> + <property name="message" value="Use System.getProperties() to get system properties."/> + </module> + + <module name="Regexp"> + <property name="format" value="Long\.getLong"/> + <property name="illegalPattern" value="true"/> + <property name="message" value="Use System.getProperties() to get system properties."/> + </module> + + <!-- + + IllegalImport cannot blacklist classes so we have to fall back to Regexp. + + --> + + <!-- forbid use of commons lang validate --> + <module name="Regexp"> + <property name="format" value="org\.apache\.commons\.lang3\.Validate"/> + <property name="illegalPattern" value="true"/> + <property name="message" + value="Use Guava Checks instead of Commons Validate. Please refer to the coding guidelines."/> + </module> + <module name="Regexp"> + <property name="format" value="org\.apache\.commons\.lang\."/> + <property name="illegalPattern" value="true"/> + <property name="message" value="Use commons-lang3 instead of commons-lang."/> + </module> + <module name="Regexp"> + <property name="format" value="org\.codehaus\.jettison"/> + <property name="illegalPattern" value="true"/> + <property name="message" value="Use flink-shaded-jackson instead of jettison."/> + </module> + <module name="Regexp"> + <property name="format" value="org\.testcontainers\.shaded"/> + <property name="illegalPattern" value="true"/> + <property name="message" + value="Use utilities from appropriate library instead of org.testcontainers."/> + </module> + + <!-- Enforce Java-style array declarations --> + <module name="ArrayTypeStyle"/> + + <module name="TodoComment"> + <!-- Checks that disallowed strings are not used in comments. --> + <property name="format" value="(FIXME)|(XXX)"/> + </module> + + <!-- + + IMPORT CHECKS + + --> + + <module name="RedundantImport"> + <!-- Checks for redundant import statements. --> + <property name="severity" value="error"/> + <message key="import.redundancy" + value="Redundant import {0}."/> + </module> + + <module name="IllegalImport"> + <property name="illegalPkgs" + value="autovalue.shaded, avro.shaded, com.google.api.client.repackaged, com.google.appengine.repackaged"/> + </module> + <module name="IllegalImport"> + <property name="illegalPkgs" value="com.fasterxml.jackson"/> + <message key="import.illegal" value="{0}; Use flink-shaded-jackson instead."/> + </module> + <module name="IllegalImport"> + <property name="illegalPkgs" value="org.codehaus.jackson"/> + <message key="import.illegal" value="{0}; Use flink-shaded-jackson instead."/> + </module> + <module name="IllegalImport"> + <property name="illegalPkgs" value="org.objectweb.asm"/> + <message key="import.illegal" value="{0}; Use flink-shaded-asm instead."/> + </module> + <module name="IllegalImport"> + <property name="illegalPkgs" value="io.netty"/> + <message key="import.illegal" value="{0}; Use flink-shaded-netty instead."/> + </module> + <module name="IllegalImport"> + <property name="illegalPkgs" value="com.google.common"/> + <message key="import.illegal" value="{0}; Use flink-shaded-guava instead."/> + </module> + + <module name="RedundantModifier"> + <!-- Checks for redundant modifiers on various symbol definitions. + See: http://checkstyle.sourceforge.net/config_modifier.html#RedundantModifier + + We exclude METHOD_DEF to allow final methods in final classes to make them more future-proof. + --> + <property name="tokens" + value="VARIABLE_DEF, ANNOTATION_FIELD_DEF, INTERFACE_DEF, CLASS_DEF, ENUM_DEF"/> + </module> + + <!-- + IllegalImport cannot blacklist classes, and c.g.api.client.util is used for some shaded + code and some useful code. So we need to fall back to Regexp. + --> + <module name="RegexpSinglelineJava"> + <property name="format" value="^import com.google.common.base.Preconditions;$"/> + <property name="message" value="Static import functions from Guava Preconditions"/> + </module> + + <module name="UnusedImports"> + <property name="severity" value="error"/> + <property name="processJavadoc" value="true"/> + <message key="import.unused" + value="Unused import: {0}."/> + </module> + + <!-- + + JAVADOC CHECKS + + --> + + <!-- Checks for Javadoc comments. --> + <!-- See http://checkstyle.sf.net/config_javadoc.html --> + <module name="JavadocMethod"> + <property name="scope" value="protected"/> + <property name="severity" value="error"/> + <property name="allowMissingParamTags" value="true"/> + <property name="allowMissingReturnTag" value="true"/> + </module> + + <module name="MissingJavadocMethodCheck"> + <property name="allowMissingPropertyJavadoc" value="false"/> + </module> + + <module name="MissingJavadocType"> + <property name="scope" value="protected"/> + </module> + + <!-- Check that paragraph tags are used correctly in Javadoc. --> + <module name="JavadocParagraph"/> + + <module name="JavadocType"> + <property name="authorFormat" value=".+"/> + <property name="scope" value="protected"/> + <property name="severity" value="error"/> + <property name="allowMissingParamTags" value="true"/> + </module> + + <module name="JavadocStyle"> + <property name="severity" value="error"/> + <property name="checkHtml" value="true"/> + </module> + + <module name="JavadocBlockTagLocation"/> + + <!-- + + NAMING CHECKS + + --> + + <!-- Item 38 - Adhere to generally accepted naming conventions --> + + <module name="PackageName"> + <!-- Validates identifiers for package names against the + supplied expression. --> + <!-- Here the default checkstyle rule restricts package name parts to + seven characters, this is not in line with common practice at Google. + --> + <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]{1,})*$"/> + <property name="severity" value="error"/> + </module> + + <module name="TypeNameCheck"> + <!-- Validates static, final fields against the + expression "^[A-Z][a-zA-Z0-9]*$". --> + <metadata name="altname" value="TypeName"/> + <property name="severity" value="error"/> + </module> + + <module name="ConstantNameCheck"> + <!-- Validates non-private, static, final fields against the supplied + public/package final fields "^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$". --> + <metadata name="altname" value="ConstantName"/> + <property name="applyToPublic" value="true"/> + <property name="applyToProtected" value="true"/> + <property name="applyToPackage" value="true"/> + <property name="applyToPrivate" value="true"/> + <property name="format" value="^([A-Z][A-Z0-9]*(_[A-Z0-9]+)*|FLAG_.*)$"/> + <message key="name.invalidPattern" + value="Variable ''{0}'' should be in ALL_CAPS (if it is a constant) or be private (otherwise)."/> + <property name="severity" value="error"/> + </module> + + <module name="StaticVariableNameCheck"> + <!-- Validates static, non-final fields against the supplied + expression "^[a-z][a-zA-Z0-9]*_?$". --> + <metadata name="altname" value="StaticVariableName"/> + <property name="applyToPublic" value="true"/> + <property name="applyToProtected" value="true"/> + <property name="applyToPackage" value="true"/> + <property name="applyToPrivate" value="true"/> + <property name="format" value="^[a-z][a-zA-Z0-9]*_?$"/> + <property name="severity" value="error"/> + </module> + + <module name="MemberNameCheck"> + <!-- Validates non-static members against the supplied expression. --> + <metadata name="altname" value="MemberName"/> + <property name="applyToPublic" value="true"/> + <property name="applyToProtected" value="true"/> + <property name="applyToPackage" value="true"/> + <property name="applyToPrivate" value="true"/> + <property name="format" value="^[a-z][a-zA-Z0-9]*$"/> + <property name="severity" value="error"/> + </module> + + <module name="MethodNameCheck"> + <!-- Validates identifiers for method names. --> + <metadata name="altname" value="MethodName"/> + <property name="format" value="^[a-z][a-zA-Z0-9]*(_[a-zA-Z0-9]+)*$"/> + <property name="severity" value="error"/> + </module> + + <module name="ParameterName"> + <!-- Validates identifiers for method parameters against the + expression "^[a-z][a-zA-Z0-9]*$". --> + <property name="severity" value="error"/> + </module> + + <module name="LocalFinalVariableName"> + <!-- Validates identifiers for local final variables against the + expression "^[a-z][a-zA-Z0-9]*$". --> + <property name="severity" value="error"/> + </module> + + <module name="LocalVariableName"> + <!-- Validates identifiers for local variables against the + expression "^[a-z][a-zA-Z0-9]*$". --> + <property name="severity" value="error"/> + </module> + + <!-- + + LENGTH and CODING CHECKS + + --> + + <!-- Checks for braces around if and else blocks --> + <module name="NeedBraces"> + <property name="severity" value="error"/> + <property name="tokens" + value="LITERAL_IF, LITERAL_ELSE, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO"/> + </module> + + <module name="UpperEll"> + <!-- Checks that long constants are defined with an upper ell.--> + <property name="severity" value="error"/> + </module> + + <module name="FallThrough"> + <!-- Warn about falling through to the next case statement. Similar to + javac -Xlint:fallthrough, but the check is suppressed if a single-line comment + on the last non-blank line preceding the fallen-into case contains 'fall through' (or + some other variants that we don't publicized to promote consistency). + --> + <property name="reliefPattern" + value="fall through|Fall through|fallthru|Fallthru|falls through|Falls through|fallthrough|Fallthrough|No break|NO break|no break|continue on"/> + <property name="severity" value="error"/> + </module> + + <!-- Checks for over-complicated boolean expressions. --> + <module name="SimplifyBooleanExpression"/> + + <!-- Detects empty statements (standalone ";" semicolon). --> + <module name="EmptyStatement"/> + + <!-- Detect multiple consecutive semicolons (e.g. ";;"). --> + <module name="RegexpSinglelineJava"> + <property name="format" value=";{2,}"/> + <property name="message" value="Use one semicolon"/> + <property name="ignoreComments" value="true"/> + </module> + + <!-- + + MODIFIERS CHECKS + + --> + + <module name="ModifierOrder"> + <!-- Warn if modifier order is inconsistent with JLS3 8.1.1, 8.3.1, and + 8.4.3. The prescribed order is: + public, protected, private, abstract, static, final, transient, volatile, + synchronized, native, strictfp + --> + <property name="severity" value="error"/> + </module> + + + <!-- + + WHITESPACE CHECKS + + --> + + <module name="EmptyLineSeparator"> + <!-- Checks for empty line separator between tokens. The only + excluded token is VARIABLE_DEF, allowing class fields to + be declared on consecutive lines. + --> + <property name="allowMultipleEmptyLines" value="false"/> + <property name="allowMultipleEmptyLinesInsideClassMembers" value="false"/> + <property name="tokens" value="PACKAGE_DEF, IMPORT, STATIC_IMPORT, CLASS_DEF, + INTERFACE_DEF, ENUM_DEF, STATIC_INIT, INSTANCE_INIT, METHOD_DEF, + CTOR_DEF"/> + </module> + + <module name="SingleSpaceSeparator"/> + + <module name="WhitespaceAround"> + <!-- Checks that various tokens are surrounded by whitespace. + This includes most binary operators and keywords followed + by regular or curly braces. + --> + <property name="tokens" value="ASSIGN, BAND, BAND_ASSIGN, BOR, + BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, + EQUAL, GE, GT, LAMBDA, LAND, LE, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, + LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN, + LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS, + MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION, + SL, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/> + <property name="severity" value="error"/> + </module> + + <module name="WhitespaceAfter"> + <!-- Checks that commas, semicolons and typecasts are followed by + whitespace. + --> + <property name="tokens" value="COMMA, SEMI, TYPECAST"/> + </module> + + <module name="NoWhitespaceAfter"> + <!-- Checks that there is no whitespace after various unary operators. + Linebreaks are allowed. + --> + <property name="tokens" value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS, + UNARY_PLUS"/> + <property name="allowLineBreaks" value="true"/> + <property name="severity" value="error"/> + </module> + + <module name="NoWhitespaceBefore"> + <!-- Checks that there is no whitespace before various unary operators. + Linebreaks are allowed. + --> + <property name="tokens" value="SEMI, DOT, POST_DEC, POST_INC"/> + <property name="allowLineBreaks" value="true"/> + <property name="severity" value="error"/> + </module> + + <module name="OperatorWrap"> + <!-- Checks that assignment operators are at the end of the line. --> + <property name="option" value="eol"/> + <property name="tokens" value="ASSIGN"/> + </module> + + <module name="ParenPad"> + <!-- Checks that there is no whitespace before close parens or after + open parens. + --> + <property name="severity" value="error"/> + </module> + + </module> +</module> + diff --git a/develop/spotbugs-exclude.xml b/develop/spotbugs-exclude.xml new file mode 100644 index 00000000..5fd588b2 --- /dev/null +++ b/develop/spotbugs-exclude.xml @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8"?> +<FindBugsFilter> + +</FindBugsFilter> diff --git a/develop/suppressions.xml b/develop/suppressions.xml new file mode 100644 index 00000000..9c4d424d --- /dev/null +++ b/develop/suppressions.xml @@ -0,0 +1,12 @@ +<?xml version="1.0"?> +<!DOCTYPE suppressions PUBLIC + "-//Puppy Crawl//DTD Suppressions 1.1//EN" + "http://www.puppycrawl.com/dtds/suppressions_1_1.dtd"> + +<suppressions> + <!-- target directory is not relevant for checkstyle --> + <suppress + files="[\\/]target[\\/]" + checks=".*"/> + +</suppressions> diff --git a/docker/Dockerfile b/docker/Dockerfile index 58b0b660..b36dbbf2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o xjar . FROM ${JDK_IMAGE} MAINTAINER Galaxy VOLUME /tmp -WORKDIR /home/tsg/galaxy/galaxy-qgw-service +WORKDIR /opt/galaxy-qgw-service COPY --from=builder /build . COPY config config COPY dat dat @@ -21,4 +21,4 @@ ENV JAVA_OPTS=" -Xms1024m -Xmx2048m " ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 #ENV TZ=Asia/Almaty #RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -ENTRYPOINT [ "sh", "-c", "./xjar java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED -jar galaxy-qgw-service.xjar" ]
\ No newline at end of file +ENTRYPOINT [ "sh", "-c", "./xjar java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED --add-modules java.se --add-exports java.base/jdk.internal.ref=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.management/sun.management=ALL-UNNAMED --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED -jar galaxy-qgw-service.xjar" ]
\ No newline at end of file diff --git a/docs/release/release-340.md b/docs/release/2021/release-340.md index bdd00e07..bdd00e07 100644 --- a/docs/release/release-340.md +++ b/docs/release/2021/release-340.md diff --git a/docs/release/release-341.md b/docs/release/2021/release-341.md index e02967a0..e02967a0 100644 --- a/docs/release/release-341.md +++ b/docs/release/2021/release-341.md diff --git a/docs/release/release-342.md b/docs/release/2021/release-342.md index ff4621d7..ff4621d7 100644 --- a/docs/release/release-342.md +++ b/docs/release/2021/release-342.md diff --git a/docs/release/release-343.md b/docs/release/2021/release-343.md index 58cda80d..58cda80d 100644 --- a/docs/release/release-343.md +++ b/docs/release/2021/release-343.md diff --git a/docs/release/release-344.md b/docs/release/2021/release-344.md index 74bed113..74bed113 100644 --- a/docs/release/release-344.md +++ b/docs/release/2021/release-344.md diff --git a/docs/release/release-345.md b/docs/release/2021/release-345.md index d532596e..d532596e 100644 --- a/docs/release/release-345.md +++ b/docs/release/2021/release-345.md diff --git a/docs/release/release-346.md b/docs/release/2021/release-346.md index db9bc831..db9bc831 100644 --- a/docs/release/release-346.md +++ b/docs/release/2021/release-346.md diff --git a/docs/release/release-347.md b/docs/release/2021/release-347.md index 5838d4d5..5838d4d5 100644 --- a/docs/release/release-347.md +++ b/docs/release/2021/release-347.md diff --git a/docs/release/release-348.md b/docs/release/2021/release-348.md index aa735cb0..aa735cb0 100644 --- a/docs/release/release-348.md +++ b/docs/release/2021/release-348.md diff --git a/docs/release/release-349.md b/docs/release/2021/release-349.md index 88e5ce7f..88e5ce7f 100644 --- a/docs/release/release-349.md +++ b/docs/release/2021/release-349.md diff --git a/docs/release/release-350.md b/docs/release/2021/release-350.md index 4f8cd4cf..4f8cd4cf 100644 --- a/docs/release/release-350.md +++ b/docs/release/2021/release-350.md diff --git a/docs/release/release-v3.3-rc3.md b/docs/release/2021/release-v3.3-rc3.md index 1728eaa7..1728eaa7 100644 --- a/docs/release/release-v3.3-rc3.md +++ b/docs/release/2021/release-v3.3-rc3.md diff --git a/docs/release/release-351.md b/docs/release/2022/release-351.md index eeeec747..eeeec747 100644 --- a/docs/release/release-351.md +++ b/docs/release/2022/release-351.md diff --git a/docs/release/release-352.md b/docs/release/2022/release-352.md index 69566c32..69566c32 100644 --- a/docs/release/release-352.md +++ b/docs/release/2022/release-352.md diff --git a/docs/release/release-353.md b/docs/release/2022/release-353.md index c5432f59..c5432f59 100644 --- a/docs/release/release-353.md +++ b/docs/release/2022/release-353.md diff --git a/docs/release/release-354.md b/docs/release/2022/release-354.md index 58d4df31..58d4df31 100644 --- a/docs/release/release-354.md +++ b/docs/release/2022/release-354.md diff --git a/docs/release/release-355.md b/docs/release/2022/release-355.md index ef32beee..ef32beee 100644 --- a/docs/release/release-355.md +++ b/docs/release/2022/release-355.md diff --git a/docs/release/release-356.md b/docs/release/2022/release-356.md index 2ff94f99..2ff94f99 100644 --- a/docs/release/release-356.md +++ b/docs/release/2022/release-356.md diff --git a/docs/release/release-357.md b/docs/release/2022/release-357.md index cad662c4..cad662c4 100644 --- a/docs/release/release-357.md +++ b/docs/release/2022/release-357.md diff --git a/docs/release/release-358.md b/docs/release/2022/release-358.md index eba51cc9..eba51cc9 100644 --- a/docs/release/release-358.md +++ b/docs/release/2022/release-358.md diff --git a/docs/release/release-359.md b/docs/release/2022/release-359.md index 583e5963..583e5963 100644 --- a/docs/release/release-359.md +++ b/docs/release/2022/release-359.md diff --git a/docs/release/release-360.md b/docs/release/2022/release-360.md index a99bc12a..a99bc12a 100644 --- a/docs/release/release-360.md +++ b/docs/release/2022/release-360.md diff --git a/docs/release/release-361.md b/docs/release/2022/release-361.md index 14bf2c70..14bf2c70 100644 --- a/docs/release/release-361.md +++ b/docs/release/2022/release-361.md diff --git a/docs/release/release-362.md b/docs/release/2022/release-362.md index 6ba8f935..6ba8f935 100644 --- a/docs/release/release-362.md +++ b/docs/release/2022/release-362.md diff --git a/docs/release/release-363.md b/docs/release/2023/release-363.md index ceb661c1..ceb661c1 100644 --- a/docs/release/release-363.md +++ b/docs/release/2023/release-363.md diff --git a/docs/release/release-364.md b/docs/release/2023/release-364.md index 073d80cd..073d80cd 100644 --- a/docs/release/release-364.md +++ b/docs/release/2023/release-364.md diff --git a/docs/release/release-365.md b/docs/release/2023/release-365.md index 82020af3..82020af3 100644 --- a/docs/release/release-365.md +++ b/docs/release/2023/release-365.md diff --git a/docs/release/release-366.md b/docs/release/2023/release-366.md index 8928978a..8928978a 100644 --- a/docs/release/release-366.md +++ b/docs/release/2023/release-366.md diff --git a/docs/release/release-367.md b/docs/release/2023/release-367.md index 22da5df0..22da5df0 100644 --- a/docs/release/release-367.md +++ b/docs/release/2023/release-367.md diff --git a/docs/release/release-368.md b/docs/release/2023/release-368.md index f7a4c013..f7a4c013 100644 --- a/docs/release/release-368.md +++ b/docs/release/2023/release-368.md diff --git a/docs/release/release-369.md b/docs/release/2023/release-369.md index c4bb35c5..c4bb35c5 100644 --- a/docs/release/release-369.md +++ b/docs/release/2023/release-369.md diff --git a/docs/release/release-370.md b/docs/release/2023/release-370.md index 20aad732..20aad732 100644 --- a/docs/release/release-370.md +++ b/docs/release/2023/release-370.md diff --git a/docs/release/release-371.md b/docs/release/2023/release-371.md index dd6efe5e..dd6efe5e 100644 --- a/docs/release/release-371.md +++ b/docs/release/2023/release-371.md diff --git a/docs/release/release-372.md b/docs/release/2023/release-372.md index b8b034c0..b8b034c0 100644 --- a/docs/release/release-372.md +++ b/docs/release/2023/release-372.md diff --git a/docs/release/release-373.md b/docs/release/2023/release-373.md index c88fc839..c88fc839 100644 --- a/docs/release/release-373.md +++ b/docs/release/2023/release-373.md diff --git a/docs/release/2023/release-374.md b/docs/release/2023/release-374.md new file mode 100644 index 00000000..70400d72 --- /dev/null +++ b/docs/release/2023/release-374.md @@ -0,0 +1,45 @@ +Release 374 (TSG-23.12) +* 新增自定义聚合函数：MEDIAN_HDR、QUANTILE_HDR、PERCENTILES_HDR、COUNT_DISTINCT、APPROX_COUNT_DISTINCT_HLLD、RATE(GAL-44、TSG-18048) +* 新增自定义毫秒时间函数：FROM_UNIXTIME_MILLIS、UNIX_TIMESTAMP_MILLIS(GAL-449) +* 新增HOS API, 支持文件上传、删除、文件元信息获取接口(TSG-17843、TSG-18698、TSG-18808) +* 新增Monitor Schema，监测策略规则命中统计Schema(TSG-17851) +* 新增Trace ID，日志中新增TraceID(同响应request_id)便于日志追踪(TSG-17851) +* 删除过期Schema：radius_record、gtpc_record、radius_onff_log、sys_packet_capture_event、ctive_defence_event(TSG-17239、TSG-18119) +* 删除ES组件应用及相关代码(GAL-439) +* 重构Query API接口：管理SQL、DSL查询接口，支持批量获取(TSG-17269) +* 重构Datasets API接口，初始化内置数据集(TSG-17787) +* 重构Database API、Troubleshooting API(TSG-17843) +* 重构Util API (TSG-17843) +* 重构Knowledge Base API,对知识库文件管理接口进行重构(TSG-17843) +* 修改liveChart，移除liveChart summary uniq_client_ip指标 +* 修改Dataset，field-discovery，filter增加 vsys_id过滤条件(TSG-18229) +* 修改dataset，数monitor-event-timeline 只提供会话指标趋势(TSG-17952) +* 修改配置，Hbase Saved Query结果表为tsg_galaxy:saved_query_result +* 修改配置，HOS存储桶重命名为knowledge_base_bucket(GAL-453) +* 修改Schema，适配DosEvent、ProxyEvent与AssessmentEvent结构变化(TSG-17819) +* 修改Schema，基础字段app_path、protocol_path重命名(TSG-17828) +* 修改Schema，修改文件路径补全对应HOS桶名 +* 修改Schema，更新fqdn_category_list字段dict_location数据字典引用位置 +* 修改Schema，，删除radius_match、gtpc_match 函数(TSG-17813) +* 修改Schema，ssl_sni、dtls_sni、http_host、quic_sni字段新增sub_domain、get_value函数 +* 修改Schema，日志Schema部分字段补全Combine函数定义(TSG-17239) +* 修改Schema，logicalType、constraints.type区分timestamp、unix_timestamp +* 修改Schema，完善Log/Metric Schema 函数、Unit约束(TSG-18048) +* 修改Schema，Proxy Event decoded_as 增加DoH协议字典 +* 修改Schema，更新流量Top类统计schema metric字典 +* 修改Schema，修改 metrics schema dimensions、metrics、filters自定义报告约束 +* 修改Schema，Monitor Events中未展示Monitor Mirrored相关字段(TSG-18602) +* 修改Schema，Manipulation日志中应隐藏FQDN Category List、HTTP.Proxy Flags等字段(TSG-18577) +* 修改Schema，修正traffic_shaping_rule_hits.doc中字段名(TSG-18639) +* 扩展Schema，histogram支持MEDIAN_HDR函数 +* 扩展Schema，schema doc 增加time分类(TSG-17875) +* 扩展Schema，新增公共字段ip_protocol，支持report统计(TSG-18203) +* 扩展Schema，新增公共字段statistics_rule_list(TSG-18328) +* 扩展Schema，VoIP Record decoded_as 增加VoIP字典值(TSG-18445) +* 扩展Schema，日志Schema支持表达式字段(TSG-18511) +* 扩展Schema，object_statistics 新增object_type字段及字典数据(TSG-17960) +* 扩展Schema，描述可作为维度的时间字段(TSG-18662) +* 优化样例数据接口，优化Chart预览样例数据的生成方式(TSG-18645、TSG-18673) + + + diff --git a/docs/release/release-374.md b/docs/release/release-374.md deleted file mode 100644 index 723ff0d3..00000000 --- a/docs/release/release-374.md +++ /dev/null @@ -1,2 +0,0 @@ -Release 374 (TSG-23.12) -*
\ No newline at end of file diff --git a/docs/release/release-375.md b/docs/release/release-375.md new file mode 100644 index 00000000..4e4dbd1f --- /dev/null +++ b/docs/release/release-375.md @@ -0,0 +1,12 @@ +Release 375 (TSG-24.02) +* 修改Schema，修改部分Metric聚合及延迟粒度(TSG-18857) +* 修改Schema，Security Rule Hits Action数据字典去除 Monitor(TSG-18885) +* 修改Schema，字段tunnels重命名为encapsulation(TSG-19249) +* 修正字段发现统计结果查询失败(TSG-19354) +* 修改Schema，解决Encapsulation内容重复显示的问题(TSG-19347) +* 修改Schema，移除format function定义(GAL-488) +* 完善Schema校验接口(TSG-18390) +* 优化TIME_FLOOR_WITH_FILL函数转义(TSG-19375) +* 修改Dockerfile，更新默认路径为/opt/galaxy-qgw-service(GAL-502) +* 修改Schema，修正Schema部分字段数据类型(TSG-19671) +* 修改Schema，aggregated_metric_unit补充dns_response_latency_ms，ssl_handshake_latency_ms字段(TSG-19592)
\ No newline at end of file diff --git a/docs/release/release-376.md b/docs/release/release-376.md new file mode 100644 index 00000000..55215bdc --- /dev/null +++ b/docs/release/release-376.md @@ -0,0 +1,11 @@ +Release 376 (TSG-24.03) +* 隐藏Top Metrics Schema metric列，Dataset不再按照Metric定义不同数据源(TSG-19563) +* 优化Dry Run查询的样例数据生成方式(TSG-19467) +* 完善子查询优化判断逻辑，补充自定义聚合(TSG-19921) +* 修复样例数据，修复结果维度列数据重复问题(TSG-19890) +* 修复样例数据，修复别名存在特殊字符导致JSONPath解析失败问题(TSG-19870) +* 修复样例数据，修复时序数据多维度情况下数据不完整问题(TSG-19892) +* 扩展Schema，在tunnel_type 描述GRE(TSG-19518) +* 扩展Schema，扩展表达式字段：internal_port_list, external_port_list, interal_asn_list, external_asn_list(TSG-19579) +* 完善查询ClickHouse，以适配时区差异(GAL-520) +* 完善任务提交，针对Saved Query Job不做判重逻辑(TSG-19990)
\ No newline at end of file diff --git a/docs/release/release-377.md b/docs/release/release-377.md new file mode 100644 index 00000000..f1e905b6 --- /dev/null +++ b/docs/release/release-377.md @@ -0,0 +1,29 @@ +Release 377 (TSG-24.04) +* 新增Schema，新增Dos Protection Rule Hits Schema 和 dataset(TSG-19777) +* 新增Schema，新增DoS Protection Rule Metric Schema(TSG-19845) +* 扩展Schema，Statistics Rule Metric Schema 和 dataset 增加client_asn 和 server_asn(TSG-19744) +* 扩展Schema，Statistics Rule Metric Schema 和 dataset 增加client_country 和 server_countryTSG-19886) +* 扩展Schema，增加公共字段tunnel_id_list(TSG-20333) +* 扩展Schema，TopK Schema 维度增加rank_by(TSG-20020) +* 扩展Schema，新增基于Client/Server Geolocation公共字段(TSG-20205) +* 扩展RATE函数，支持不同时间粒度(TSG-19773) +* 修改Query API，exec_mode 修改为 execution_mode、formatType修改为output_mode(TSG-19957) +* 修改Query API，完善字段Meta类型与结果值类型不一致问题、Meta移除Category(TSG-19957) +* 修改Query API，规范request_id、job_id 为标准UUID(TSG-19957) +* 修改Dataset，时间范围变量统一命名为start_time，end_time(TSG-19957) +* 修改Schema，TopK Schema及Dataset适配修改(TSG-20020) +* 修改Schema，DoS Event 增加DoS Protection Rule相关字段(TSG-20103) +* 修改Schema，修改dup_traffic_flag、ssl_esni_flag、ssl_ech_flag数据字典(TSG-20198) +* 修改Schema，Proxy Manipulation日志隐藏server_os_desc、client_os_desc字段(TSG-20210) +* 删除Schema，删除Top Subscriber Schema及Dataset(TSG-20020) +* 删除过期代码，删除非结构化数据文件接口、实时数据分析接口、liveChart冗余代码(TSG-20173) +* 优化pom依赖，删除有冲突的无效依赖(GAL-512) +* 优化RATE函数，结果保留两位小数(TSG-20131) +* 优化Dataset，使用RATE函数替代SubQuery 嵌套(TSG-19957) +* 扩展Schema，完善Dos Events attack_type数据字典(TSG-20551) +* 扩展Schema，Schema新增mail_starttls_flag字段(TSG-20751) +* 修改Schema，address_type 数据字典调整(TSG-20589) +* 修改Database API，解决公共Schema更新未即时生效的问题(GAL-540) +* 修改Dataset，修正proxy-manipulation-policy-rule-hits-by-action 数据集(TSG-20544) +* 修改Schema，Proxy Intercept/Manipulation 隐藏 tunnel_id_list、tunnel_endpoint_a_desc、tunnel_endpoint_b_desc字段(TSG-20675) +* 优化储存配额接口，优化Get Storage quota 接口，druid、xxl-job数据不一致导致数据为空问题 diff --git a/docs/release/release-378.md b/docs/release/release-378.md new file mode 100644 index 00000000..a1813d2f --- /dev/null +++ b/docs/release/release-378.md @@ -0,0 +1,14 @@ +Release 378 (TSG-24.05) +* 新增Schema, 新增路径数据遥测日志datapath_telemetry_record(TSG-20473) +* 新增Dataset, 新增数据路径遥测相关业务数据集: count、list、packet combine(TSG-20473) +* 新增Pcapng合并功能, 扩展DSL接口支持Packet Combine合并功能(TSG-20473) +* 扩展Schema, 新增app_extra_info公共字段(TSG-20783) +* 扩展Schema, Statistics Rule 新增 c2s/s2c_ttl、c2s/s2c_link_id 字段，支持Statistics模版维度(TSG-20725) +* 扩展Schema, 日志新增公共字段c2s_ttl、s2c_ttl(TSG-20811) +* 扩展Schema, Link ID支持 Report Filter +* 扩展Schema, 新增公共字段direction(TSG-20925) +* 扩展操作符函数, 支持IS NULL、IS NOT NULL检索(TSG-20925) +* 修改Schema, VoIP Record删除与SIP/RTP协议无关字段(TSG-20765) +* 修改Schema, 设置表达式_rule_id 字段不支持日志检索 +* 修改HOS API, 参数命名遵循下划线规范(TSG-20730) +* 修改schema, 安全事件日志隐藏monitor_mirrored_pkts，monitor_mirrored_bytes字段 diff --git a/docs/release/release-379.md b/docs/release/release-379.md new file mode 100644 index 00000000..952ebfb1 --- /dev/null +++ b/docs/release/release-379.md @@ -0,0 +1,12 @@ +Release 379 (TSG-24.06) +* 扩展Schema, 新增公共字段app_category(TSG-21392) +* 新增Schema, 新增流量透视Metric traffic_sketch_metric(TSG-20989) +* 新增Dataset, 新增Traffic Spectrum Dataset(TSG-20989) +* 扩展DSL接口, 支持Traffic Spectrum相关功能(TSG-20989) +* 修正Schema, 修正Packets 相关字段统计单位(TSG-21471) +* 调整HDR_GET_PERCENTILES percentileTicksPerHalfDistance参数默认值为5 (TSG-21466) +* 适配通过nginx查询clickhouse(GAL-585) +* 优化TopN优化最内层SQL，Limit下限制设置为1000(TSG-21485) +* 优化Drilldown查询，结果数据标识Subtotal rows(TSG-21477) +* 完善字段发现，修正Array类型字段发现未统计空记录(TSG-21480) +* 修改Schema，flags Asymmetric 标注已过期 Deprecated
\ No newline at end of file diff --git a/docs/release/release-380.md b/docs/release/release-380.md new file mode 100644 index 00000000..b12c499d --- /dev/null +++ b/docs/release/release-380.md @@ -0,0 +1,13 @@ +Release 380 (TSG-24.07) +* 新增储存配额 sys_storage_event表(TSG-21555) +* 扩展Schema，IP类型字段支持 isIPAddressInRange(TSG-21544) +* 扩展Schema，Statistics Rule Metric 增加IP Fragments字段(TSG-21637) +* 扩展Schema，数据路径遥测日志新增字段egress_action(TSG-21768) +* 扩展时间粒度函数，DSL、SQL Query Job支持默认时间粒度(TSG-21420) +* 扩展Function，转义isIPAddressInRange，解决空值导致函数执行异常问题 +* 扩展Traffic Spectrum 二部图，支持时间分片查询 +* 修正Traffic Spectrum Throughput 流量曲线粒度(TSG-21648) +* 完善样例数据，避免schema dict值数量小于随机结果数量时，生成数据异常问题 +* 完善审计日志，查询网关具备记录高延迟查询 SQL 的功能(GAL-603) +* 优化储存配额接口，新增max_usage、基于mariadb获取数据、删除冗余代码(TSG-21554) +* 优化Schema，统一命名:Client-to-Server、Server-to-Client(TSG-21651)
\ No newline at end of file @@ -5,21 +5,21 @@ <modelVersion>4.0.0</modelVersion> <groupId>com.mesalab</groupId> <artifactId>galaxy-qgw-service</artifactId> - <version>367</version> + <version>380</version> <packaging>jar</packaging> <name>galaxy-qgw-service</name> <repositories> <repository> - <id>nexus</id> + <id>nexus3</id> <name>Team Nexus Repository</name> - <url>http://192.168.40.153:8099/content/groups/public</url> + <url>http://192.168.40.153:8081/repository/public/</url> </repository> </repositories> <pluginRepositories> <pluginRepository> - <id>nexus</id> + <id>nexus3</id> <name>Team Nexus Repository</name> - <url>http://192.168.40.153:8099/content/groups/public</url> + <url>http://192.168.40.153:8081/repository/public/</url> </pluginRepository> </pluginRepositories> @@ -40,6 +40,8 @@ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <java.version>11</java.version> + <maven.compiler.source>11</maven.compiler.source> + <maven.compiler.target>11</maven.compiler.target> <starter.druid.version>1.1.22</starter.druid.version> <mysql-connector-java.version>8.0.18</mysql-connector-java.version> <commons.lang3.version>3.5</commons.lang3.version> @@ -50,31 +52,41 @@ <commons.codec.version>1.11</commons.codec.version> <commons.configuration.version>1.10</commons.configuration.version> <commons.logging.version>1.2</commons.logging.version> - <guava.version>23.0</guava.version> <nutz.version>1.r.65</nutz.version> <log4j2.version>2.17.0</log4j2.version> <httpclient.version>4.5.6</httpclient.version> - <avro.version>1.9.1</avro.version> <active.record.version>4.9.01</active.record.version> - <json.schema.version>2.2.10</json.schema.version> - <zdjizhi.version>1.2</zdjizhi.version> + <guava.version>33.0.0-jre</guava.version> + <galaxy.tool.version>2.0.1</galaxy.tool.version> <uaAnalyser.version>1.0-RELEASE</uaAnalyser.version> <jsqlparser.version>4.6</jsqlparser.version> <hbase.client.version>2.2.3</hbase.client.version> <hutool.version>5.6.1</hutool.version> <fastjson2.version>2.0.32</fastjson2.version> <json.path.version>2.5.0</json.path.version> - <docker.build>192.168.40.153</docker.build> <docker.build.port>2375</docker.build.port> <docker.registry>192.168.40.153</docker.registry> - <docker.registry.port>9080</docker.registry.port> - <docker.image.prefix>tsg/galaxy</docker.image.prefix> + <docker.registry.port>8082</docker.registry.port> + <docker.image.prefix>tsg/olap</docker.image.prefix> <nacos.config.version>0.2.7</nacos.config.version> <spotless.version>2.30.0</spotless.version> + <hazelcast-spring.version>5.3.6</hazelcast-spring.version> </properties> + <profiles> + <profile> + <id>java11</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <properties> + <maven.compiler.source>11</maven.compiler.source> + <maven.compiler.target>11</maven.compiler.target> + </properties> + </profile> + </profiles> - <dependencies> + <dependencies> <dependency> <groupId>com.mesalab.ua</groupId> @@ -109,12 +121,24 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-actuator</artifactId> </dependency> + <!-- https://mvnrepository.com/artifact/org.springframework.cloud/spring-cloud-starter-sleuth --> + <dependency> + <groupId>org.springframework.cloud</groupId> + <artifactId>spring-cloud-starter-sleuth</artifactId> + <version>2.2.7.RELEASE</version> + </dependency> + <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>${mysql-connector-java.version}</version> </dependency> <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>${guava.version}</version> + </dependency> + <dependency> <groupId>io.micrometer</groupId> <artifactId>micrometer-core</artifactId> </dependency> @@ -152,6 +176,12 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> + <exclusions> + <exclusion> + <artifactId>json-path</artifactId> + <groupId>com.jayway.jsonpath</groupId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.commons</groupId> @@ -200,11 +230,6 @@ <version>${commons.logging.version}</version> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>${guava.version}</version> - </dependency> - <dependency> <groupId>org.nutz</groupId> <artifactId>nutz</artifactId> <version>${nutz.version}</version> @@ -213,6 +238,12 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>${httpclient.version}</version> + <exclusions> + <exclusion> + <artifactId>commons-codec</artifactId> + <groupId>commons-codec</groupId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.github.jsqlparser</groupId> @@ -220,11 +251,6 @@ <version>${jsqlparser.version}</version> </dependency> <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - <version>${avro.version}</version> - </dependency> - <dependency> <groupId>com.zdjizhi</groupId> <artifactId>shade-calcite</artifactId> <version>1.0.0</version> @@ -232,12 +258,28 @@ <dependency> <groupId>com.geedgenetworks</groupId> <artifactId>galaxy</artifactId> - <version>${zdjizhi.version}</version> + <version>${galaxy.tool.version}</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> </exclusion> + <exclusion> + <artifactId>log4j</artifactId> + <groupId>log4j</groupId> + </exclusion> + <exclusion> + <artifactId>commons-lang3</artifactId> + <groupId>org.apache.commons</groupId> + </exclusion> + <exclusion> + <artifactId>commons-io</artifactId> + <groupId>commons-io</groupId> + </exclusion> + <exclusion> + <artifactId>commons-codec</artifactId> + <groupId>commons-codec</groupId> + </exclusion> </exclusions> </dependency> <dependency> @@ -246,11 +288,6 @@ <version>${hutool.version}</version> </dependency> <dependency> - <groupId>com.github.java-json-tools</groupId> - <artifactId>json-schema-validator</artifactId> - <version>${json.schema.version}</version> - </dependency> - <dependency> <groupId>com.alibaba.fastjson2</groupId> <artifactId>fastjson2</artifactId> <version>${fastjson2.version}</version> @@ -269,6 +306,24 @@ <groupId>com.alibaba.boot</groupId> <artifactId>nacos-config-spring-boot-starter</artifactId> <version>${nacos.config.version}</version> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <artifactId>simpleclient</artifactId> + <groupId>io.prometheus</groupId> + </exclusion> + <exclusion> + <artifactId>commons-lang3</artifactId> + <groupId>org.apache.commons</groupId> + </exclusion> + <exclusion> + <artifactId>commons-codec</artifactId> + <groupId>commons-codec</groupId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.phoenix</groupId> @@ -279,6 +334,58 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> </exclusion> + <exclusion> + <artifactId>avro</artifactId> + <groupId>org.apache.avro</groupId> + </exclusion> + <exclusion> + <artifactId>guava</artifactId> + <groupId>com.google.guava</groupId> + </exclusion> + <exclusion> + <artifactId>zookeeper</artifactId> + <groupId>org.apache.zookeeper</groupId> + </exclusion> + <exclusion> + <artifactId>netty</artifactId> + <groupId>io.netty</groupId> + </exclusion> + <exclusion> + <artifactId>jsr305</artifactId> + <groupId>com.google.code.findbugs</groupId> + </exclusion> + <exclusion> + <artifactId>joda-time</artifactId> + <groupId>joda-time</groupId> + </exclusion> + <exclusion> + <artifactId>error_prone_annotations</artifactId> + <groupId>com.google.errorprone</groupId> + </exclusion> + <exclusion> + <artifactId>commons-io</artifactId> + <groupId>commons-io</groupId> + </exclusion> + <exclusion> + <artifactId>commons-configuration</artifactId> + <groupId>commons-configuration</groupId> + </exclusion> + <exclusion> + <artifactId>commons-collections4</artifactId> + <groupId>org.apache.commons</groupId> + </exclusion> + <exclusion> + <artifactId>commons-codec</artifactId> + <groupId>commons-codec</groupId> + </exclusion> + <exclusion> + <artifactId>checker-qual</artifactId> + <groupId>org.checkerframework</groupId> + </exclusion> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> </exclusions> </dependency> <dependency> @@ -301,6 +408,31 @@ <artifactId>flyway-core</artifactId> <version>5.2.4</version> </dependency> + <dependency> + <groupId>com.hazelcast</groupId> + <artifactId>hazelcast</artifactId> + <version>${hazelcast-spring.version}</version> + </dependency> + <dependency> + <groupId>com.opencsv</groupId> + <artifactId>opencsv</artifactId> + <version>5.7.1</version> + <exclusions> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + <exclusion> + <artifactId>commons-lang3</artifactId> + <groupId>org.apache.commons</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.networknt</groupId> + <artifactId>json-schema-validator</artifactId> + <version>1.3.3</version> + </dependency> </dependencies> <build> @@ -330,6 +462,28 @@ </plugin> <plugin> + <groupId>org.commonjava.maven.plugins</groupId> + <artifactId>directory-maven-plugin</artifactId> + <executions> + <execution> + <id>directories</id> + <goals> + <goal>directory-of</goal> + </goals> + <phase>initialize</phase> + <configuration> + <property>rootDir</property> + <project> + <groupId>com.mesalab</groupId> + <artifactId>galaxy-qgw-service</artifactId> + </project> + </configuration> + </execution> + </executions> + </plugin> + + + <plugin> <groupId>org.gaul</groupId> <artifactId>modernizer-maven-plugin</artifactId> <version>2.1.0</version> @@ -351,29 +505,16 @@ <plugin> <groupId>com.github.spotbugs</groupId> <artifactId>spotbugs-maven-plugin</artifactId> - <version>4.0.0</version> + <version>4.4.2.2</version> <configuration> <skip>true</skip> - <jvmArgs>-Xmx1024m</jvmArgs> + <threshold>Low</threshold> + <effort>default</effort> <failOnError>true</failOnError> + <xmlOutput>true</xmlOutput> + <excludeFilterFile>${qgw.main.basedir}/develop/spotbugs-exclude.xml</excludeFilterFile> + <spotbugsXmlOutputDirectory>${project.build.directory}/spotbugs</spotbugsXmlOutputDirectory> </configuration> - <executions> - <execution> - <id>default</id> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> - <dependencies> - <!-- overwrite dependency on spotbugs if you want to specify the version of spotbugs --> - <dependency> - <groupId>com.github.spotbugs</groupId> - <artifactId>spotbugs</artifactId> - <version>4.0.0</version> - </dependency> - </dependencies> </plugin> <plugin> @@ -429,32 +570,30 @@ <plugin> <groupId>org.jacoco</groupId> <artifactId>jacoco-maven-plugin</artifactId> - <version>0.8.5</version> + <version>0.8.7</version> <executions> <execution> - <id>default</id> + <id>default-agent</id> <goals> <goal>prepare-agent</goal> </goals> + <phase>initialize</phase> </execution> <execution> - <id>report</id> - <phase>prepare-package</phase> + <id>default-report</id> + <phase>verify</phase> <goals> <goal>report</goal> </goals> </execution> </executions> - <configuration> - <skip>false</skip> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> - <version>3.1.1</version> + <version>3.1.2</version> <configuration> <configLocation>google_checks.xml</configLocation> <encoding>UTF-8</encoding> @@ -468,7 +607,7 @@ <dependency> <groupId>com.puppycrawl.tools</groupId> <artifactId>checkstyle</artifactId> - <version>8.32</version> + <version>8.40</version> </dependency> </dependencies> <executions> @@ -480,9 +619,15 @@ </goals> </execution> </executions> + </plugin> + <plugin> + <groupId>org.sonarsource.scanner.maven</groupId> + <artifactId>sonar-maven-plugin</artifactId> + <version>3.9.0.2155</version> </plugin> + </plugins> </pluginManagement> @@ -521,10 +666,10 @@ <password>Geedge2020!</password> <excludes> <exclude> - resources/** + META-INF/resources/** </exclude> <exclude> - META-INF/resources/** + resources/** </exclude> </excludes> </configuration> @@ -536,7 +681,29 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-checkstyle-plugin</artifactId> - <version>3.1.1</version> + <executions> + <execution> + <id>java-style-check</id> + <phase>compile</phase> + <goals> + <goal>check</goal> + </goals> + <configuration> + <sourceDirectories>src/main/java</sourceDirectories> + </configuration> + </execution> + <execution> + <id>java-test-style-check</id> + <phase>test-compile</phase> + <goals> + <goal>check</goal> + </goals> + <configuration> + <testSourceDirectories>src/test/java</testSourceDirectories> + <includeTestSourceDirectory>true</includeTestSourceDirectory> + </configuration> + </execution> + </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -547,12 +714,25 @@ <plugin> <groupId>com.github.spotbugs</groupId> <artifactId>spotbugs-maven-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.jacoco</groupId> - <artifactId>jacoco-maven-plugin</artifactId> - <version>0.8.5</version> + <executions> + <execution> + <id>findbugs-main</id> + <phase>compile</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + <execution> + <id>findbugs-test</id> + <phase>test-compile</phase> + <goals> + <goal>check</goal> + </goals> + <configuration> + <includeTests>true</includeTests> + </configuration> + </execution> + </executions> </plugin> <plugin> @@ -575,7 +755,7 @@ <artifactId>docker-maven-plugin</artifactId> <version>1.0.0</version> <configuration> - <serverId>153-docker-repo</serverId> + <serverId>153-nexus-docker-repo</serverId> <registryUrl>${docker.registry}:${docker.registry.port}</registryUrl> <pushImage>true</pushImage> <forceTags>true</forceTags> @@ -612,31 +792,21 @@ </resource> </resources> </configuration> + <dependencies> + <dependency> + <groupId>javax.activation</groupId> + <artifactId>activation</artifactId> + <version>1.1.1</version> + </dependency> + </dependencies> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-toolchains-plugin</artifactId> - <version>1.1</version> - <executions> - <execution> - <goals> - <goal>toolchain</goal> - </goals> - </execution> - </executions> - <configuration> - <toolchains> - <jdk> - <version>11</version> - <vendor>amazon</vendor> - </jdk> - </toolchains> - </configuration> - </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-resources-plugin</artifactId> + <version>3.2.0</version> <configuration> <encoding>utf-8</encoding> <useDefaultDelimiters>true</useDefaultDelimiters> diff --git a/src/main/java/com/mesalab/calcite/CalciteMemoryUtils.java b/src/main/java/com/mesalab/calcite/CalciteConnectionProcessor.java index 6c7fadc4..48c95f7b 100644 --- a/src/main/java/com/mesalab/calcite/CalciteMemoryUtils.java +++ b/src/main/java/com/mesalab/calcite/CalciteConnectionProcessor.java @@ -1,7 +1,5 @@ package com.mesalab.calcite; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -9,6 +7,7 @@ import com.mesalab.calcite.storage.DataTypeMapping; import com.mesalab.calcite.storage.Storage; import com.mesalab.common.exception.BusinessException; import com.zdjizhi.calcite.jdbc.CalciteConnection; +import lombok.extern.slf4j.Slf4j; import java.io.File; import java.io.IOException; @@ -18,12 +17,13 @@ import java.util.*; import static com.geedgenetworks.utils.StringUtil.setDefaultIfEmpty; -public class CalciteMemoryUtils { - private static final Log log = LogFactory.get(); +@Slf4j +public class CalciteConnectionProcessor { private static final String URL_PREFIX = "jdbc:calcite:model="; private static final String URL_SUFFIX = "config" + File.separator + "memorySchema.json"; - + private CalciteConnectionProcessor() { + } static { try { Class.forName("com.zdjizhi.calcite.jdbc.Driver"); @@ -32,8 +32,22 @@ public class CalciteMemoryUtils { } } - public static Map<String, Object> executeMemoryQuery(String tableName, List<Map<String, String>> meta, + private static class CalciteQueryHolder { + private static final CalciteConnectionProcessor INSTANCE = new CalciteConnectionProcessor(); + } + + public static CalciteConnectionProcessor getInstance() { + return CalciteQueryHolder.INSTANCE; + } + + + + public Map<String, Object> executeMemoryQuery(String tableName, List<Map<String, String>> meta, List<Map<String, Object>> data, String sql) { + CalciteConnection connection = null; + ResultSet resultSet = null; + PreparedStatement statement = null; + Map<String, Object> result = Maps.newHashMap(); if (meta.isEmpty() || data.isEmpty()) { data.clear(); @@ -42,9 +56,8 @@ public class CalciteMemoryUtils { result.put("data", data); return result; } - ResultSet resultSet = null; - PreparedStatement statement = null; - CalciteConnection connection = null; + + try { connection = getConnection().unwrap(CalciteConnection.class); Storage.DummyTable table = buildDummyTable(meta, tableName); @@ -89,7 +102,7 @@ public class CalciteMemoryUtils { } - private static Connection getConnection() throws IOException, SQLException { + private Connection getConnection() throws IOException, SQLException { String url = URL_PREFIX + new File("").getCanonicalPath() + File.separator + URL_SUFFIX; Properties info = new Properties(); info.setProperty("lex", "MYSQL_ANSI"); @@ -101,7 +114,7 @@ public class CalciteMemoryUtils { - private static void close(ResultSet rs, PreparedStatement pst, Connection conn) { + private void close(ResultSet rs, PreparedStatement pst, Connection conn) { if (rs != null) { try { rs.close(); @@ -136,7 +149,7 @@ public class CalciteMemoryUtils { * @param data 原始结果集合 * @return */ - private static Storage.DummyTable loadData(Storage.DummyTable table, List<Map<String, Object>> data) { + private Storage.DummyTable loadData(Storage.DummyTable table, List<Map<String, Object>> data) { List<Storage.DummyColumn> columns = table.getColumns(); data.stream().forEach(item -> { List list = Lists.newArrayList(); @@ -164,7 +177,7 @@ public class CalciteMemoryUtils { * @param tableName 表名 * @return */ - private static Storage.DummyTable buildDummyTable(List<Map<String, String>> meta, String tableName) { + private Storage.DummyTable buildDummyTable(List<Map<String, String>> meta, String tableName) { Storage.DummyTable dummyTable = new Storage.DummyTable(tableName); for (Map<String, String> column : meta) { String fieldName = column.get("name"); @@ -183,7 +196,7 @@ public class CalciteMemoryUtils { return dummyTable; } - private static final String intUnits = "Int8,Int16,Int32,Int64,UInt8,UInt16,UInt32,UInt64,bigint,int,long"; - private static final String doubleUnits = "Float32,Float64,float,double"; + private final String intUnits = "Int8,Int16,Int32,Int64,UInt8,UInt16,UInt32,UInt64,bigint,int,long"; + private final String doubleUnits = "Float32,Float64,float,double"; } diff --git a/src/main/java/com/mesalab/cn/component/DslValidate.java b/src/main/java/com/mesalab/cn/component/DslValidate.java index 4ba5d96b..c357bc42 100644 --- a/src/main/java/com/mesalab/cn/component/DslValidate.java +++ b/src/main/java/com/mesalab/cn/component/DslValidate.java @@ -3,12 +3,11 @@ package com.mesalab.cn.component; import com.geedgenetworks.utils.StringUtil; import com.mesalab.cn.entity.pojo.DSLObject; import com.mesalab.cn.enums.*; -import com.mesalab.cn.exception.CNErrorCode; import com.mesalab.cn.exception.CNErrorMessage; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; -import com.mesalab.qgw.service.MetadataService; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.service.DatabaseService; import org.apache.commons.lang3.EnumUtils; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -34,80 +33,67 @@ public class DslValidate { public static final Pattern strFormatDateTime = Pattern.compile("\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}", Pattern.CASE_INSENSITIVE); @Autowired - private MetadataService metadataService; + private DatabaseService databaseService; public void executeValidate(DSLObject dslObject) throws BusinessException { if (StringUtil.isEmpty(dslObject)) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.DSL_OBJECT_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.DSL_OBJECT_IS_INVALID)); } DSLObject.Query query = dslObject.getQuery(); if (StringUtil.isEmpty(query)) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.QUERY_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.QUERY_IS_INVALID)); } if (StringUtil.isEmpty(EngineTypeEnum.getByEngine(query.getDataEngine()))) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.DATA_ENGINE_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.DATA_ENGINE_IS_INVALID)); } if (ANALYSIS_ENGINE.getEngine().equals(query.getDataEngine())) { if (StringUtil.isBlank(query.getDataSource())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.DATASOURCE_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.DATASOURCE_IS_INVALID)); } } else { - if (StringUtil.isBlank(metadataService.getDBTypeByTableName(query.getDataSource()))) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.DATASOURCE_IS_INVALID)); + if (StringUtil.isBlank(databaseService.getDBEngineByTableName(query.getDataSource()))) { + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.DATASOURCE_IS_INVALID)); } } DSLObject.Query.Parameters parameters = query.getParameters(); if (StringUtil.isEmpty(parameters)) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.PARAMETERS_IS_INVALID)); - } - if (!isValidMatch(parameters.getMatch())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.MATCH_IS_INVALID)); - } - if (!isValidRange(parameters.getRange())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.RANGE_TYPE_IS_INVALID)); - } - if (!isValidDistinct(parameters.getDistinct())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.DISTINCT_TYPE_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_IS_INVALID)); } + validMatch(parameters.getMatch()); + validRange(parameters.getRange()); + validDistinct(parameters.getDistinct()); if (!isValidIntervals(parameters.getIntervals())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.INTERVALS_TYPE_IS_INVALID)); - } - if (!isValidSort(parameters.getSort())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.SORT_TYPE_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.INTERVALS_TYPE_IS_INVALID)); } + validSort(parameters.getSort()); } /** * 校验sort: - * 1.是否属于{@link RangeTypeEnum}限定类型 + * 通过: 属于{@link RangeTypeEnum}限定类型 * * @param sort * @return */ - private boolean isValidSort(List<DSLObject.Query.Parameters.Sort> sort) { + private void validSort(List<DSLObject.Query.Parameters.Sort> sort) { if (CollectionUtils.isEmpty(sort)) { - return true; + return; } sort.forEach(o -> { if (!StringUtil.isBlank(o.getType())) { if (StringUtil.isEmpty(SortTypeEnum.getByType(StringUtil.lowerCase(o.getType())))) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.SORT_TYPE_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.SORT_TYPE_IS_INVALID)); } } }); - return true; } /** @@ -129,19 +115,19 @@ public class DslValidate { } for (String dateTimeStr : split) { if (!strFormatDateTime.matcher(dateTimeStr).find()) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.TIME_FORMAT_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.TIME_FORMAT_ERROR)); } } try { DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"); if (dateTimeFormatter.parseMillis(split[1]) < dateTimeFormatter.parseMillis(split[0])) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.INTERVALS_VALUE_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.INTERVALS_VALUE_ERROR)); } } catch (RuntimeException e) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.TIME_FORMAT_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.TIME_FORMAT_ERROR)); } return true; } @@ -149,79 +135,74 @@ public class DslValidate { /** * 校验Distinct: - * 1.是否属于{@link RangeTypeEnum}限定类型 + * 通过: 属于{@link RangeTypeEnum}限定类型 * * @param distinct * @return */ - private boolean isValidDistinct(DSLObject.Query.Parameters.Distinct distinct) { + private void validDistinct(DSLObject.Query.Parameters.Distinct distinct) { if (StringUtil.isEmpty(distinct)) { - return true; + return; } validLogicType(distinct.getLogicType()); distinct.getCondition().forEach(o -> { if (StringUtil.isNotBlank(o.getType())) { - if (!EnumUtils.isValidEnum(RangeTypeEnum.class, StringUtil.upperCase(o.getType()))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.DISTINCT_TYPE_ERROR)); + if (!EnumUtils.isValidEnum(RangeTypeEnum.class, StringUtil.upperCase(o.getType()))) { } } }); - return true; } /** * 校验range: - * 1.是否属于{@link RangeTypeEnum}限定类型 + * 通过: 属于{@link RangeTypeEnum}限定类型 * * @param range * @return */ - private boolean isValidRange(DSLObject.Query.Parameters.Range range) { + private void validRange(DSLObject.Query.Parameters.Range range) { if (StringUtil.isEmpty(range)) { - return true; + return; } validLogicType(range.getLogicType()); range.getCondition().forEach(o -> { if (StringUtil.isNotBlank(o.getType())) { - if (!EnumUtils.isValidEnum(RangeTypeEnum.class, StringUtil.upperCase(o.getType()))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.RANGE_TYPE_ERROR)); + if (!EnumUtils.isValidEnum(RangeTypeEnum.class, StringUtil.upperCase(o.getType()))) { + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.RANGE_TYPE_ERROR)); } } }); - return true; } /** * 校验match: - * 1.是否属于{@link MatchTypeEnum}限定类型 + * 通过: 属于{@link MatchTypeEnum}限定类型 * * @param match * @return */ - private boolean isValidMatch(DSLObject.Query.Parameters.Match match) { + private void validMatch(DSLObject.Query.Parameters.Match match) { if (StringUtil.isEmpty(match)) { - return true; + return; } String logicType = match.getLogicType(); validLogicType(logicType); match.getCondition().forEach(o -> { if (StringUtil.isNotBlank(o.getType())) { - if (!EnumUtils.isValidEnum(MatchTypeEnum.class, StringUtil.upperCase(o.getType()))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.MATCH_TYPE_ERROR)); + if (!EnumUtils.isValidEnum(MatchTypeEnum.class, StringUtil.upperCase(o.getType()))) { + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.MATCH_TYPE_ERROR)); } } }); - return true; } private void validLogicType(String logicType) { if (StringUtil.isNotBlank(logicType)) { if (StringUtil.isEmpty(LogicTypeEnum.getByType(StringUtil.lowerCase(logicType)))) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),CNErrorMessage.LOGIC_TYPE_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.LOGIC_TYPE_IS_INVALID)); } } } diff --git a/src/main/java/com/mesalab/cn/entity/pojo/CNParser.java b/src/main/java/com/mesalab/cn/entity/pojo/CNParser.java index 128aef7c..008d53b6 100644 --- a/src/main/java/com/mesalab/cn/entity/pojo/CNParser.java +++ b/src/main/java/com/mesalab/cn/entity/pojo/CNParser.java @@ -4,12 +4,10 @@ import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.geedgenetworks.utils.StringUtil; import com.google.common.base.Joiner; -import com.mesalab.cn.constant.ErrorMessage; import com.mesalab.cn.constant.SqlKeywords; import com.mesalab.cn.enums.RangeTypeEnum; import com.mesalab.cn.exception.CNErrorCode; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -43,7 +41,7 @@ public class CNParser extends DSLParser { selectSql = parseQueryOfShareNet(dslObject, selectSql); } catch (RuntimeException e) { log.error("Build SQL Exception:{}",e); - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), CNErrorCode.SQL_BUILDER_EXCEPTION.getCode(), + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CNErrorCode.SQL_BUILDER_EXCEPTION.getCode(), String.format(CNErrorCode.SQL_BUILDER_EXCEPTION.getMessage(),e.getMessage())); } return selectSql; @@ -58,7 +56,7 @@ public class CNParser extends DSLParser { selectSql = parseQueryOfBannedApp(dslObject, selectSql); } catch (RuntimeException e) { log.error("Build SQL Exception:{}",e); - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), CNErrorCode.SQL_BUILDER_EXCEPTION.getCode(), + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CNErrorCode.SQL_BUILDER_EXCEPTION.getCode(), String.format(CNErrorCode.SQL_BUILDER_EXCEPTION.getMessage(),e.getMessage())); } return selectSql; diff --git a/src/main/java/com/mesalab/cn/service/impl/CNServiceImpl.java b/src/main/java/com/mesalab/cn/service/impl/CNServiceImpl.java index 3e7ea389..22958c1b 100644 --- a/src/main/java/com/mesalab/cn/service/impl/CNServiceImpl.java +++ b/src/main/java/com/mesalab/cn/service/impl/CNServiceImpl.java @@ -5,8 +5,8 @@ import cn.hutool.log.LogFactory; import com.geedgenetworks.utils.StringUtil; import com.mesalab.cn.entity.pojo.*; import com.mesalab.cn.service.CNService; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.service.QueryService; +import com.mesalab.qgw.model.basic.SQLQueryContext; +import com.mesalab.qgw.service.SQLSyncQueryService; import com.mesalab.ua.analyser.UaAnalyser; import com.mesalab.ua.analyser.enums.UserInfo; import com.mesalab.ua.analyser.impl.UaAnalyserImpl; @@ -36,15 +36,15 @@ public class CNServiceImpl implements CNService { private CNParser cnParser; @Autowired - private QueryService queryService; + private SQLSyncQueryService sqlSyncQueryService; @Override public BaseResult shareNetQuery(DSLObject dslObject) throws BusinessException { dslObject.getQuery().getParameters().setGranularity("user_id,ip"); String executeSql = cnParser.buildShareNetSQL(dslObject); - QueryProfile queryProfile = new QueryProfile(); - queryProfile.setQuery(executeSql); - BaseResult baseResult = queryService.executeQuery(queryProfile); + SQLQueryContext queryProfile = new SQLQueryContext(); + queryProfile.setOriginalSQL(executeSql); + BaseResult baseResult = sqlSyncQueryService.executeQuery(queryProfile); if (!baseResult.isSuccess()) { return baseResult; } @@ -68,9 +68,7 @@ public class CNServiceImpl implements CNService { public BaseResult bannedAppQuery(DSLObject dslObject) throws BusinessException { dslObject.getQuery().getParameters().setGranularity("user_id,ip"); String executeSql = cnParser.bannedAppSQL(dslObject); - QueryProfile queryProfile = new QueryProfile(); - queryProfile.setQuery(executeSql); - BaseResult baseResult = queryService.executeQuery(queryProfile); + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(executeSql).build()); if (!baseResult.isSuccess()) { return baseResult; } diff --git a/src/main/java/com/mesalab/cn/service/impl/EntityDetailServiceImpl.java b/src/main/java/com/mesalab/cn/service/impl/EntityDetailServiceImpl.java index f3d7e17a..f7f4c3aa 100644 --- a/src/main/java/com/mesalab/cn/service/impl/EntityDetailServiceImpl.java +++ b/src/main/java/com/mesalab/cn/service/impl/EntityDetailServiceImpl.java @@ -13,15 +13,14 @@ import com.mesalab.cn.entity.pojo.Condition; import com.mesalab.cn.entity.pojo.DSLObject; import com.mesalab.cn.entity.pojo.DSLParser; import com.mesalab.cn.entity.EntityDetail; -import com.mesalab.cn.exception.CNErrorCode; import com.mesalab.cn.exception.CNErrorMessage; import com.mesalab.cn.service.EntityDetailService; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; import com.mesalab.knowledge.strategy.BaseQueryProvider; +import com.mesalab.common.exception.CommonErrorCode; import org.apache.commons.compress.utils.Lists; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -511,29 +510,29 @@ public class EntityDetailServiceImpl implements EntityDetailService { private void valid(DSLObject.Query.Parameters parameters) { DSLObject.Query.Parameters.Match match = parameters.getMatch(); if (ObjectUtil.isEmpty(match)) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_IS_INVALID)); } List<Condition> condition = match.getCondition(); - if (ObjectUtil.isEmpty(condition)) { throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_MATCH_CONDITION_IS_INVALID)); + if (ObjectUtil.isEmpty(condition)) { throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_MATCH_CONDITION_IS_INVALID)); } if (ObjectUtil.isEmpty(condition.get(0).getFieldValues())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_MATCH_CONDITION_FIELD_VALUE_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_MATCH_CONDITION_FIELD_VALUE_IS_INVALID)); } if (ObjectUtil.isEmpty(condition.get(0).getFieldKey())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_MATCH_CONDITION_FIELD_KEY_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_MATCH_CONDITION_FIELD_KEY_IS_INVALID)); } if (!(Arrays.asList(APP_NAME, DOMAIN, IP, DNS).contains(condition.get(0).getFieldKey()))) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_MATCH_CONDITION_FIELD_KEY_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), CNErrorMessage.PARAMETERS_MATCH_CONDITION_FIELD_KEY_IS_INVALID)); } Integer limit = parameters.getLimit(); if(ObjectUtil.isNotEmpty(limit)&&!(limit>=0&&limit<=DEF_LIMIT)){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), String.format(CNErrorMessage.PARAMETERS_LIMIT_IS_INVALID),DEF_LIMIT)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), String.format(CNErrorMessage.PARAMETERS_LIMIT_IS_INVALID),DEF_LIMIT)); } } private boolean existServerIp(String ip){ diff --git a/src/main/java/com/mesalab/common/configuration/HazelcastConfiguration.java b/src/main/java/com/mesalab/common/configuration/HazelcastConfiguration.java new file mode 100644 index 00000000..29e38c85 --- /dev/null +++ b/src/main/java/com/mesalab/common/configuration/HazelcastConfiguration.java @@ -0,0 +1,35 @@ +package com.mesalab.common.configuration; + +import com.hazelcast.config.Config; +import com.hazelcast.config.YamlConfigBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; + +/** + * TODO + * + * @Classname HazelcastConfig + * @Date 2023/12/9 10:01 + * @Author wWei + */ +@Configuration +public class HazelcastConfiguration { + @Bean + public Config hazelcastConfig() throws IOException, URISyntaxException { + Resource resource = new FileSystemResource(new File("").getCanonicalPath() + File.separator + "config" + File.separator + "hazelcast.yaml"); + if (!resource.exists()) { + resource = new ClassPathResource("hazelcast.yaml"); + } + InputStream inputStream = resource.getInputStream(); + YamlConfigBuilder configBuilder = new YamlConfigBuilder(inputStream); + return configBuilder.build(); + } +} diff --git a/src/main/java/com/mesalab/common/configuration/HeavyResourceThreadPoolConfiguration.java b/src/main/java/com/mesalab/common/configuration/HeavyResourceThreadPoolConfiguration.java new file mode 100644 index 00000000..d9a004cf --- /dev/null +++ b/src/main/java/com/mesalab/common/configuration/HeavyResourceThreadPoolConfiguration.java @@ -0,0 +1,44 @@ +package com.mesalab.common.configuration; + + +import com.mesalab.services.configuration.JobConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * TODO + * + * @Classname HeavyResourceThreadPoolConfiguration + * @Date 2021/12/1 9:20 上午 + * @Author wWei + */ +@Configuration +@EnableAsync +public class HeavyResourceThreadPoolConfiguration { + private static final String THREAD_NAME_PREFIX = "Heavy-Resource-ThreadPool-"; + + private JobConfig jobCfg; + + @Bean("heavyResourceThreadPool") + public ThreadPoolTaskExecutor getHeavyResourceThreadPool() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(jobCfg.getHeavyResourceCorePoolSize()); + executor.setMaxPoolSize(jobCfg.getHeavyResourceMaxPoolSize()); + executor.setQueueCapacity(jobCfg.getHeavyResourceQueueCapacity()); + executor.setKeepAliveSeconds(10); + executor.setThreadNamePrefix(THREAD_NAME_PREFIX); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); + executor.initialize(); + return executor; + } + + @Autowired + public void setJobConfig(JobConfig jobCfg) { + this.jobCfg = jobCfg; + } +}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/common/configuration/LightWeightThreadPoolConfiguration.java b/src/main/java/com/mesalab/common/configuration/LightWeightThreadPoolConfiguration.java new file mode 100644 index 00000000..95cba49b --- /dev/null +++ b/src/main/java/com/mesalab/common/configuration/LightWeightThreadPoolConfiguration.java @@ -0,0 +1,46 @@ +package com.mesalab.common.configuration; + +import com.mesalab.services.configuration.JobConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * TODO + * + * @Classname lightWeightThreadPoolConfiguration + * @Date 2021/12/1 9:20 上午 + * @Author wWei + */ +@Configuration +@EnableAsync +public class LightWeightThreadPoolConfiguration { + + private JobConfig jobConfig; + private static final String THREAD_NAME_PREFIX = "Light-Weight-ThreadPool-"; + + @Bean("lightWeightThreadPool") + public ThreadPoolTaskExecutor getLightWeightThreadPool() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(jobConfig.getLightWeightCorePoolSize()); + executor.setMaxPoolSize(jobConfig.getLightWeightMaxPoolSize()); + executor.setQueueCapacity(jobConfig.getLightWeightQueueCapacity()); + executor.setKeepAliveSeconds(10); + executor.setThreadNamePrefix(THREAD_NAME_PREFIX); + + // 线程池对拒绝任务的处理策略 + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); + // 初始化 + executor.initialize(); + return executor; + } + + @Autowired + private void setJobConfig(JobConfig jobConfig) { + this.jobConfig = jobConfig; + } +}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/services/configuration/TaskThreadPoolCfg.java b/src/main/java/com/mesalab/common/configuration/TaskThreadPoolConfiguration.java index 8affab52..aa6d3524 100644 --- a/src/main/java/com/mesalab/services/configuration/TaskThreadPoolCfg.java +++ b/src/main/java/com/mesalab/common/configuration/TaskThreadPoolConfiguration.java @@ -1,5 +1,7 @@ -package com.mesalab.services.configuration; +package com.mesalab.common.configuration; +import com.mesalab.services.configuration.JobConfig; +import com.mesalab.services.configuration.TaskConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -11,28 +13,31 @@ import java.util.concurrent.ThreadPoolExecutor; /** * TODO * - * @Classname ThreadPoolCfgSubJob - * @Date 2021/12/3 9:13 上午 + * @Classname TaskThreadPoolConfig + * @Date 2024/6/26 10:56 * @Author wWei */ @Configuration @EnableAsync -public class TaskThreadPoolCfg { - @Autowired - TaskConfig taskConfig; - private static final int keepAliveTime = 10; - private static final String threadNamePrefix = "Async(sub)-Service-"; +public class TaskThreadPoolConfiguration { + private TaskConfig taskConfig; + private static final String THREAD_NAME_PREFIX = "Task-ThreadPool-"; - @Bean("taskExecutor") + @Bean("taskThreadPool") public ThreadPoolTaskExecutor getAsyncExecutor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(taskConfig.getPoolCorePoolSize()); executor.setMaxPoolSize(taskConfig.getPoolMaxPoolSize()); executor.setQueueCapacity(taskConfig.getPoolQueueCapacity()); - executor.setKeepAliveSeconds(keepAliveTime); - executor.setThreadNamePrefix(threadNamePrefix); + executor.setKeepAliveSeconds(10); + executor.setThreadNamePrefix(THREAD_NAME_PREFIX); executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); executor.initialize(); return executor; } + + @Autowired + private void setTaskConfig(TaskConfig taskConfig) { + this.taskConfig = taskConfig; + } } diff --git a/src/main/java/com/mesalab/common/entity/BaseResult.java b/src/main/java/com/mesalab/common/entity/BaseResult.java index 6f4cfd20..1cff317b 100644 --- a/src/main/java/com/mesalab/common/entity/BaseResult.java +++ b/src/main/java/com/mesalab/common/entity/BaseResult.java @@ -1,6 +1,8 @@ package com.mesalab.common.entity; -import com.mesalab.common.enums.ResultStatusEnum; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.mesalab.common.enums.HttpStatusCodeEnum; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -20,15 +22,17 @@ import java.util.Map; @Builder @NoArgsConstructor @AllArgsConstructor +@JsonPropertyOrder({"status", "message", "requestId", "success", "code", "job", "output_mode", "statistics", "meta", "data"}) public class BaseResult<T> implements Serializable { private Integer status; private String code; - private String queryKey; + @JsonProperty("request_id") + private String requestId; - private boolean success; + private Boolean success; private String message; @@ -36,23 +40,25 @@ public class BaseResult<T> implements Serializable { private Map<String, Object> job; - private String formatType; + @JsonProperty("output_mode") + private String outputMode; private T meta; private T data; - public BaseResult(Integer status, String code, String queryKey, boolean success, String message, Map<String, Object> statistics, String formatType, T meta, T data) { + public BaseResult(Integer status, String code, String requestId, boolean success, String message, Map<String, Object> statistics, String outputMode, T meta, T data) { this.status = status; this.code = code; - this.queryKey = queryKey; + this.requestId = requestId; this.success = success; this.message = message; this.statistics = statistics; - this.formatType = formatType; + this.outputMode = outputMode; this.meta = meta; this.data = data; } + /** * 判断是否是成功结果 * JsonIgnore使之不在json序列化结果当中 @@ -60,7 +66,22 @@ public class BaseResult<T> implements Serializable { * @return 是否为成功结果 */ public boolean isSuccess() { - return this.status >= ResultStatusEnum.SUCCESS.getCode() && this.status < 300; + return this.status != null && this.status >= HttpStatusCodeEnum.SUCCESS.getCode() && this.status < 300; } + @Override + public String toString() { + return "BaseResult{" + + "status=" + status + + ", code='" + code + '\'' + + ", requestId='" + requestId + '\'' + + ", success=" + success + + ", message='" + message + '\'' + + ", statistics=" + statistics + + ", job=" + job + + ", output_mode='" + outputMode + '\'' + + ", meta=" + meta + + ", data=" + data + + '}'; + } }
\ No newline at end of file diff --git a/src/main/java/com/mesalab/common/entity/BaseResultGenerator.java b/src/main/java/com/mesalab/common/entity/BaseResultGenerator.java index 12caae20..ede4c628 100644 --- a/src/main/java/com/mesalab/common/entity/BaseResultGenerator.java +++ b/src/main/java/com/mesalab/common/entity/BaseResultGenerator.java @@ -2,9 +2,9 @@ package com.mesalab.common.entity; import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; +import com.mesalab.common.exception.CommonErrorCode; import java.util.Map; @@ -21,14 +21,15 @@ public class BaseResultGenerator { * * @param status 返回HTTP响应码 * @param code 返回业务编码 + * @param success 是否成功 * @param message 返回消息 * @param data 返回数据 - * @param formatType 返回数据格式 JSON/CSV + * @param outputMode 返回数据格式 json/jsonWithMetadata/csv * @param <T> 返回数据类型 * @return 返回结果 */ - public static <T> BaseResult<T> generate(final int status, final String code, final String message, T data, T meta, final Map<String, Object> statistics, final String formatType) { - return new BaseResult<>(status, code,null, false, message, statistics, formatType, meta, data); + public static <T> BaseResult<T> generate(final int status, final String code, final boolean success, final String message, T data, T meta, final Map<String, Object> statistics, final String outputMode) { + return new BaseResult<>(status, code,null, success, message, statistics, outputMode, meta, data); } /** @@ -37,7 +38,7 @@ public class BaseResultGenerator { * @return 操作成功的默认响应结果 */ public static <T> BaseResult<T> success() { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), + return new BaseResult<>(HttpStatusCodeEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, ResultCodeEnum.SUCCESS.getMessage(), null, null, null, null); } @@ -50,7 +51,7 @@ public class BaseResultGenerator { * @return 响应结果 */ public static <T> BaseResult<T> success(final String message, final T data) { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), + return new BaseResult<>(HttpStatusCodeEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, message, null, null, null, data); } @@ -78,7 +79,7 @@ public class BaseResultGenerator { * @return 响应结果 */ public static <T> BaseResult<T> success(final String message, final T data, final Map<String, Object> statistics) { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), + return new BaseResult<>(HttpStatusCodeEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, message, statistics, null, null, data); } @@ -90,7 +91,7 @@ public class BaseResultGenerator { * @return 响应结果 */ public static <T> BaseResult<T> success(final T data) { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, + return new BaseResult<>(HttpStatusCodeEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, ResultCodeEnum.SUCCESS.getMessage(), null, null, null, data); } @@ -99,14 +100,16 @@ public class BaseResultGenerator { * * @param statistics * @param job + * @param outputMode + * @param meta * @param data * @return {@link BaseResult<T>} * @created by wWei * @date 2021/11/26 11:46 上午 */ - public static <T> BaseResult<T> success(final Map<String, Object> statistics, final Map<String, Object> job, final T meta, final T data) { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, - ResultCodeEnum.SUCCESS.getMessage(), statistics, job, null, meta, data); + public static <T> BaseResult<T> success(final Map<String, Object> statistics, final Map<String, Object> job,final String outputMode, final T meta, final T data) { + return new BaseResult<>(HttpStatusCodeEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, + ResultCodeEnum.SUCCESS.getMessage(), statistics, job, outputMode, meta, data); } /** @@ -118,8 +121,8 @@ public class BaseResultGenerator { * @date 2023/11/29 11:46 上午 */ public static <T> BaseResult<T> successCreate(final Map<String, Object> job) { - return new BaseResult<>(ResultStatusEnum.CREATED.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, - ResultCodeEnum.SUCCESS.getMessage(), null, job, null, null, null); + return new BaseResult<>(HttpStatusCodeEnum.CREATED.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, + HttpStatusCodeEnum.CREATED.getMessage(), null, job, null, null, null); } @@ -130,7 +133,7 @@ public class BaseResultGenerator { * @return 响应结果 */ public static <T> BaseResult<T> success4Message(final String message) { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), + return new BaseResult<>(HttpStatusCodeEnum.SUCCESS.getCode(), ResultCodeEnum.SUCCESS.getCode(), null, true, message, null, null, null, null); } @@ -140,8 +143,8 @@ public class BaseResultGenerator { * @return 操作成功的默认响应结果 */ public static <T> BaseResult<T> failure() { - return new BaseResult<>(ResultStatusEnum.BAD_REQUEST.getCode(), - ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), null, false, ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), null, null, null, null); + return new BaseResult<>(HttpStatusCodeEnum.BAD_REQUEST.getCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), null, false, CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), null, null, null, null); } /** @@ -152,7 +155,7 @@ public class BaseResultGenerator { * @return 响应结果 */ public static <T> BaseResult<T> failure(final int status, final String message) { - return new BaseResult<>(status, ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), null, false, message, null, null, null, null); + return new BaseResult<>(status, CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), null, false, message, null, null, null, null); } /** @@ -187,7 +190,7 @@ public class BaseResultGenerator { * @return 响应结果 */ public static <T> BaseResult<T> failure(final int status, final String message, T data) { - return new BaseResult<>(status, ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), null, false, message, null, null, null, data); + return new BaseResult<>(status, CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), null, false, message, null, null, null, data); } /** @@ -196,8 +199,8 @@ public class BaseResultGenerator { * @param resultStatusEnum 自定义错误编码枚举 * @return 响应结果 */ - public static <T> BaseResult<T> failure(final ResultStatusEnum resultStatusEnum) { - return new BaseResult<>(resultStatusEnum.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), null, false, resultStatusEnum.getMessage(), null, null, null, null); + public static <T> BaseResult<T> failure(final HttpStatusCodeEnum resultStatusEnum) { + return new BaseResult<>(resultStatusEnum.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), null, false, resultStatusEnum.getMessage(), null, null, null, null); } @@ -210,7 +213,7 @@ public class BaseResultGenerator { */ public static <T> BaseResult<T> failure(final String message, final String queryKey) { - return new BaseResult<>(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), queryKey, false, message, null, null, null, null); + return new BaseResult<>(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), queryKey, false, message, null, null, null, null); } @@ -220,7 +223,7 @@ public class BaseResultGenerator { * @return 操作成功的默认响应结果 */ public static <T> BaseResult<T> error() { - return new BaseResult<>(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), null, false, ResultStatusEnum.SERVER_ERROR.getMessage(), null, null, null, null); + return new BaseResult<>(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), null, false, HttpStatusCodeEnum.SERVER_ERROR.getMessage(), null, null, null, null); } /** @@ -231,7 +234,7 @@ public class BaseResultGenerator { * @return 响应结果 */ public static <T> BaseResult<T> error(final int code, final String message) { - return new BaseResult<>(code, ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), null, false, message, null, null, null, null); + return new BaseResult<>(code, CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), null, false, message, null, null, null, null); } /** @@ -251,8 +254,8 @@ public class BaseResultGenerator { * @param resultStatusEnum 自定义错误编码枚举 * @return 响应结果 */ - public static <T> BaseResult<T> error(final ResultStatusEnum resultStatusEnum) { - return new BaseResult<>(resultStatusEnum.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), null, false, resultStatusEnum.getMessage(), null, null, null, null); + public static <T> BaseResult<T> error(final HttpStatusCodeEnum resultStatusEnum) { + return new BaseResult<>(resultStatusEnum.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), null, false, resultStatusEnum.getMessage(), null, null, null, null); } /** @@ -262,7 +265,7 @@ public class BaseResultGenerator { * @return 响应结果 */ public static <T> BaseResult<T> error(final BusinessException be) { - return new BaseResult<>(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), null, false, be.getMessage(), null, null, null, null); + return new BaseResult<>(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), null, false, be.getMessage(), null, null, null, null); } /** @@ -272,7 +275,7 @@ public class BaseResultGenerator { * @return 响应结果 */ public static <T> BaseResult<T> error(final String message) { - return new BaseResult<>(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), null, false, message, null, null, null, null); + return new BaseResult<>(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), null, false, message, null, null, null, null); } } diff --git a/src/main/java/com/mesalab/common/entity/DataTypeMapping.java b/src/main/java/com/mesalab/common/entity/DataTypeMapping.java index fea187cf..af501983 100644 --- a/src/main/java/com/mesalab/common/entity/DataTypeMapping.java +++ b/src/main/java/com/mesalab/common/entity/DataTypeMapping.java @@ -16,4 +16,5 @@ public class DataTypeMapping { public static final String DATE = "date"; public static final String TIMESTAMP = "timestamp"; public static final String BOOLEAN = "boolean"; + public static final String ARRAY = "array"; } diff --git a/src/main/java/com/mesalab/common/enums/BooleanEnum.java b/src/main/java/com/mesalab/common/enums/BooleanOption.java index bc5ee496..c4ad53e2 100644 --- a/src/main/java/com/mesalab/common/enums/BooleanEnum.java +++ b/src/main/java/com/mesalab/common/enums/BooleanOption.java @@ -9,7 +9,7 @@ import lombok.Getter; * @date 2018/5/7 */ @Getter -public enum BooleanEnum { +public enum BooleanOption { /** * YES: 1 @@ -23,7 +23,7 @@ public enum BooleanEnum { private int value; - BooleanEnum(int value) { + BooleanOption(int value) { this.value = value; } diff --git a/src/main/java/com/mesalab/common/enums/ConsistencyOption.java b/src/main/java/com/mesalab/common/enums/ConsistencyOption.java new file mode 100644 index 00000000..3d162ddc --- /dev/null +++ b/src/main/java/com/mesalab/common/enums/ConsistencyOption.java @@ -0,0 +1,13 @@ +package com.mesalab.common.enums; + +import lombok.Getter; + +@Getter +public enum ConsistencyOption { + + TTL_CONSISTENCY("ttl_consistency"), + SCHEMA_CONSISTENCY("schema_consistency"), + DATASET_VERIFICATION("dataset_verification"); + private String value; + ConsistencyOption(String value){this.value = value;} +} diff --git a/src/main/java/com/mesalab/common/enums/CookieEnum.java b/src/main/java/com/mesalab/common/enums/CookieOption.java index 898369b1..0d3b9c4f 100644 --- a/src/main/java/com/mesalab/common/enums/CookieEnum.java +++ b/src/main/java/com/mesalab/common/enums/CookieOption.java @@ -9,7 +9,7 @@ import lombok.Getter; * @date 2018/5/26 */ @Getter -public enum CookieEnum { +public enum CookieOption { /** * REMEMBER_ME： Cookie中存储的REMEMBER_ME */ @@ -17,7 +17,7 @@ public enum CookieEnum { private String value; - CookieEnum(String value) { + CookieOption(String value) { this.value = value; } } diff --git a/src/main/java/com/mesalab/common/enums/DBTypeEnum.java b/src/main/java/com/mesalab/common/enums/DBEngineType.java index e4ac8e3f..528e5144 100644 --- a/src/main/java/com/mesalab/common/enums/DBTypeEnum.java +++ b/src/main/java/com/mesalab/common/enums/DBEngineType.java @@ -3,12 +3,12 @@ package com.mesalab.common.enums; import lombok.Getter; @Getter -public enum DBTypeEnum { - ENGINE("qgw"), +public enum DBEngineType { + QGW("qgw"), CLICKHOUSE("clickHouse"), DRUID("druid"), HBASE("hbase"), ARANGODB("arangoDB"); private String value; - DBTypeEnum(String value) {this.value = value;} + DBEngineType(String value) {this.value = value;} } diff --git a/src/main/java/com/mesalab/common/enums/DiagnosisOptionEnum.java b/src/main/java/com/mesalab/common/enums/DiagnosisOption.java index 2bf22760..ccd8c191 100644 --- a/src/main/java/com/mesalab/common/enums/DiagnosisOptionEnum.java +++ b/src/main/java/com/mesalab/common/enums/DiagnosisOption.java @@ -10,7 +10,7 @@ import lombok.Getter; * @Author wWei */ @Getter -public enum DiagnosisOptionEnum { +public enum DiagnosisOption { /** * 解析SQL测试集，用于SQL语法解析 */ @@ -26,7 +26,7 @@ public enum DiagnosisOptionEnum { private String value; - DiagnosisOptionEnum(String value) { + DiagnosisOption(String value) { this.value = value; } diff --git a/src/main/java/com/mesalab/common/enums/EnvironmentEnum.java b/src/main/java/com/mesalab/common/enums/Environment.java index 421fb63e..e81657cd 100644 --- a/src/main/java/com/mesalab/common/enums/EnvironmentEnum.java +++ b/src/main/java/com/mesalab/common/enums/Environment.java @@ -9,7 +9,7 @@ import lombok.Getter; * @date 2018/5/26 */ @Getter -public enum EnvironmentEnum { +public enum Environment { /** * 开发环境 @@ -22,7 +22,7 @@ public enum EnvironmentEnum { private String name; - EnvironmentEnum(String name) { + Environment(String name) { this.name = name; } diff --git a/src/main/java/com/mesalab/common/enums/EnvironmentGroupEnum.java b/src/main/java/com/mesalab/common/enums/EnvironmentGroupEnum.java index 5e433e05..b4d1540d 100644 --- a/src/main/java/com/mesalab/common/enums/EnvironmentGroupEnum.java +++ b/src/main/java/com/mesalab/common/enums/EnvironmentGroupEnum.java @@ -16,14 +16,14 @@ public enum EnvironmentGroupEnum { * 1. DEV(开发环境) * 2. PROD(生产环境) */ - RUNTIME(new EnvironmentEnum[]{EnvironmentEnum.DEV, EnvironmentEnum.PROD}),; + RUNTIME(new Environment[]{Environment.DEV, Environment.PROD}),; /** * 运行环境 */ - private EnvironmentEnum[] environments; + private Environment[] environments; - EnvironmentGroupEnum(EnvironmentEnum[] environments) { + EnvironmentGroupEnum(Environment[] environments) { this.environments = environments; } @@ -34,8 +34,8 @@ public enum EnvironmentGroupEnum { * @return boolean */ public static boolean isRuntime(String s) { - EnvironmentEnum[] environmentEnums = RUNTIME.getEnvironments(); - for (EnvironmentEnum environmentEnum : environmentEnums) { + Environment[] environmentEnums = RUNTIME.getEnvironments(); + for (Environment environmentEnum : environmentEnums) { if (environmentEnum.getName().equals(s)) { return true; } diff --git a/src/main/java/com/mesalab/common/enums/ExecutionMode.java b/src/main/java/com/mesalab/common/enums/ExecutionMode.java new file mode 100644 index 00000000..5aebdcc7 --- /dev/null +++ b/src/main/java/com/mesalab/common/enums/ExecutionMode.java @@ -0,0 +1,32 @@ +package com.mesalab.common.enums; + +/** + * Create Query Job Execution Mode + * + * @Classname ExecutionMode + * @Date 2023/12/6 17:06 + * @Author wWei + */ +public enum ExecutionMode { + /** + * Desc: run an asynchronous query + */ + NORMAL("normal"), + /** + * Desc: return the Job ID when the job is complete (sync query) + */ + BLOCKING("blocking"), + /** + * Desc: returns results in the same call(sync query). Does not return the Job ID + */ + ONESHOT("oneshot"); + private final String value; + + ExecutionMode(String value) { + this.value = value.toLowerCase(); + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/com/mesalab/common/enums/FileCategory.java b/src/main/java/com/mesalab/common/enums/FileCategory.java new file mode 100644 index 00000000..2042e6ab --- /dev/null +++ b/src/main/java/com/mesalab/common/enums/FileCategory.java @@ -0,0 +1,15 @@ +package com.mesalab.common.enums; + +public enum FileCategory { + APP("APP"), + IP("IP"), + DOMAIN("DOMAIN"); + private String value; + FileCategory(String value){ + this.value = value; + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/com/mesalab/common/enums/FileFormat.java b/src/main/java/com/mesalab/common/enums/FileFormat.java new file mode 100644 index 00000000..6524415f --- /dev/null +++ b/src/main/java/com/mesalab/common/enums/FileFormat.java @@ -0,0 +1,16 @@ +package com.mesalab.common.enums; + +public enum FileFormat { + MMDB("MMDB"), + CSV("CSV"), + TXT("TXT"), + AES("AES"); + private String value; + FileFormat(String value){ + this.value = value; + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/com/mesalab/common/enums/ResultStatusEnum.java b/src/main/java/com/mesalab/common/enums/HttpStatusCodeEnum.java index 7bf032fc..40b0ce40 100644 --- a/src/main/java/com/mesalab/common/enums/ResultStatusEnum.java +++ b/src/main/java/com/mesalab/common/enums/HttpStatusCodeEnum.java @@ -9,7 +9,7 @@ import lombok.Getter; * @date 2018/3/21 */ @Getter -public enum ResultStatusEnum { +public enum HttpStatusCodeEnum { /** * SUCCESS: 200 成功 * BAD_REQUEST: 400 服务端无法理解该请求 @@ -17,12 +17,15 @@ public enum ResultStatusEnum { * SERVER_ERROR: 500 网络服务异常 */ SUCCESS(200, "Success"), - CREATED(201, "Create"), - ACCEPTED(202, "Accept"), + CREATED(201, "Created"), + ACCEPTED(202, "Accepted"), + NO_CONTENT(204, "No Content"), + PARTIAL_CONTENT(206, "Partial Content"), BAD_REQUEST(400, "Bad request"), NOT_FOUND(404, "Not found"), REQUEST_URI_TOO_LONG(414, "Request URI too long"), REQ_FORBIDDEN(403, "Repeat request"), + REQ_CONFLICT(409, "Conflict"), LOCKED(423,"locked"), BAD_GATEWAY(502, "Bad gateway"), SERVICE_UNAVAILABLE (503,"Service unavailable"), @@ -33,7 +36,7 @@ public enum ResultStatusEnum { private String message; - ResultStatusEnum(int code, String message) { + HttpStatusCodeEnum(int code, String message) { this.code = code; } diff --git a/src/main/java/com/mesalab/common/enums/JobHandlerEnum.java b/src/main/java/com/mesalab/common/enums/JobHandlerOption.java index 1ef1d055..ab059723 100644 --- a/src/main/java/com/mesalab/common/enums/JobHandlerEnum.java +++ b/src/main/java/com/mesalab/common/enums/JobHandlerOption.java @@ -4,7 +4,7 @@ import lombok.Getter; @Getter -public enum JobHandlerEnum { +public enum JobHandlerOption { DELETE_TRAFFIC_DATA_JOB_HANDLER("deleteTrafficDataJobHandler"), DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER("deleteReportAndMetricsDataJobHandler"), @@ -12,10 +12,11 @@ public enum JobHandlerEnum { DELETE_ALL_TRAFFIC_DATA_JOB_HANDLER("deleteAllTrafficDataJobHandler"), DELETE_ALL_REPORT_AND_METRICS_DATA_JOB_HANDLER("deleteAllReportAndMetricsDataJobHandler"), DELETE_ALL_FILES_JOB_HANDLER("deleteAllFilesJobHandler"), + DELETE_OLD_LOG_JOB_HANDLER("deleteOldLogJobHandler"), ; private String value; - JobHandlerEnum(String value) { + JobHandlerOption(String value) { this.value = value; } } diff --git a/src/main/java/com/mesalab/common/enums/MetadataTypeEnum.java b/src/main/java/com/mesalab/common/enums/MetadataType.java index fe626f1d..ac3a1250 100644 --- a/src/main/java/com/mesalab/common/enums/MetadataTypeEnum.java +++ b/src/main/java/com/mesalab/common/enums/MetadataType.java @@ -9,7 +9,7 @@ import lombok.Getter; * @Description : 元数据类型枚举 */ @Getter -public enum MetadataTypeEnum { +public enum MetadataType { TABLES("tables"), @@ -18,7 +18,7 @@ public enum MetadataTypeEnum { private String value; - MetadataTypeEnum(String value) { + MetadataType(String value) { this.value = value; } } diff --git a/src/main/java/com/mesalab/common/enums/OutputMode.java b/src/main/java/com/mesalab/common/enums/OutputMode.java new file mode 100644 index 00000000..d048845c --- /dev/null +++ b/src/main/java/com/mesalab/common/enums/OutputMode.java @@ -0,0 +1,36 @@ +package com.mesalab.common.enums; + +import lombok.Getter; + +/** + * Create Query Job Execution Mode + * + * @Classname ExecutionMode + * @Date 2023/12/6 17:06 + * @Author wWei + */ +@Getter +public enum OutputMode { + + /** + * Desc: Output data in JSON format, But no meta. + */ + JSON("json"), + /** + * Desc: Output data in CSV format. + */ + CSV("csv"), + /** + * Desc: Output data in JSON format. It also contains some meta and statistics. + */ + JSON_WITH_METADATA("jsonWithMetadata"), + /** + * Desc: Output data in JSON format, But no key. It also contains some meta and statistics. + */ + JSON_COMPACT("jsonCompact"); + private final String value; + + OutputMode(String value) { + this.value = value; + } +} diff --git a/src/main/java/com/mesalab/common/enums/QueryFormatEnum.java b/src/main/java/com/mesalab/common/enums/QueryFormatEnum.java deleted file mode 100644 index 741139ca..00000000 --- a/src/main/java/com/mesalab/common/enums/QueryFormatEnum.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.Getter; - -@Getter -public enum QueryFormatEnum { - JSON("json"), - CSV("csv"),; - private String value; - QueryFormatEnum(String value) {this.value = value;} -} diff --git a/src/main/java/com/mesalab/common/enums/QueryOptionEnum.java b/src/main/java/com/mesalab/common/enums/QueryOption.java index b10ff5a5..c8a4bd8f 100644 --- a/src/main/java/com/mesalab/common/enums/QueryOptionEnum.java +++ b/src/main/java/com/mesalab/common/enums/QueryOption.java @@ -3,12 +3,12 @@ package com.mesalab.common.enums; import lombok.Getter; @Getter -public enum QueryOptionEnum { +public enum QueryOption { REAL_TIME("real_time"), LONG_TERM("long_term"), SYNTAX_VALIDATION("syntax_validation"), SYNTAX_PARSE("syntax_parse"); private String value; - QueryOptionEnum(String value) {this.value = value;} + QueryOption(String value) {this.value = value;} } diff --git a/src/main/java/com/mesalab/common/enums/QueryParamEnum.java b/src/main/java/com/mesalab/common/enums/QueryParam.java index b9718eb5..868c7fae 100644 --- a/src/main/java/com/mesalab/common/enums/QueryParamEnum.java +++ b/src/main/java/com/mesalab/common/enums/QueryParam.java @@ -3,10 +3,10 @@ package com.mesalab.common.enums; import lombok.Getter; @Getter -public enum QueryParamEnum { +public enum QueryParam { QUERY("query"), FORMAT("format"), OPTION("option"),; private String value; - QueryParamEnum(String value) {this.value = value;} + QueryParam(String value) {this.value = value;} } diff --git a/src/main/java/com/mesalab/common/enums/ResultCodeEnum.java b/src/main/java/com/mesalab/common/enums/ResultCodeEnum.java index 8b0ac317..868aaa6f 100644 --- a/src/main/java/com/mesalab/common/enums/ResultCodeEnum.java +++ b/src/main/java/com/mesalab/common/enums/ResultCodeEnum.java @@ -10,13 +10,7 @@ import lombok.Getter; */ @Getter public enum ResultCodeEnum { - - - SUCCESS("20000666", "Success"), - // 请求参数校验错误 - PARAMETER_ERROR("40000100", "Bad request - Invalid query parameters :%s"), - // 未知异常 - UNKNOWN_EXCEPTION("50000999", "An unknown error occurred: %s"); + SUCCESS("20000666", "Success"); private String code; private String message; diff --git a/src/main/java/com/mesalab/common/enums/SessionEnum.java b/src/main/java/com/mesalab/common/enums/SessionOption.java index e475ddc8..efd3b9a8 100644 --- a/src/main/java/com/mesalab/common/enums/SessionEnum.java +++ b/src/main/java/com/mesalab/common/enums/SessionOption.java @@ -9,7 +9,7 @@ import lombok.Getter; * @date 2018/5/10 */ @Getter -public enum SessionEnum { +public enum SessionOption { /** * CURRENT_USER: session中存储的当前用户 @@ -18,7 +18,7 @@ public enum SessionEnum { private String value; - SessionEnum(String value) { + SessionOption(String value) { this.value = value; } } diff --git a/src/main/java/com/mesalab/common/exception/BusinessException.java b/src/main/java/com/mesalab/common/exception/BusinessException.java index 2c9ce537..b8f40c74 100644 --- a/src/main/java/com/mesalab/common/exception/BusinessException.java +++ b/src/main/java/com/mesalab/common/exception/BusinessException.java @@ -1,8 +1,6 @@ package com.mesalab.common.exception; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.qgw.exception.QGWErrorCode; +import com.mesalab.common.enums.HttpStatusCodeEnum; import lombok.*; /** @@ -21,12 +19,12 @@ public class BusinessException extends RuntimeException { /** * 状态码 */ - private int status = ResultStatusEnum.SERVER_ERROR.getCode(); + private int status = HttpStatusCodeEnum.SERVER_ERROR.getCode(); /** * 业务码 */ - private String code = ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(); + private String code = CommonErrorCode.UNKNOWN_EXCEPTION.getCode(); /** diff --git a/src/main/java/com/mesalab/common/exception/CommonErrorCode.java b/src/main/java/com/mesalab/common/exception/CommonErrorCode.java new file mode 100644 index 00000000..c8163573 --- /dev/null +++ b/src/main/java/com/mesalab/common/exception/CommonErrorCode.java @@ -0,0 +1,37 @@ +package com.mesalab.common.exception; + +import lombok.Getter; + +/** + * @Author wxs + * @Date 2022/11/3 + */ +@Getter +public enum CommonErrorCode { + + /** + * 1-3位：异常类型（HTTP协议状态码） + * 3-4位：模块 01 + * 5-7：自然排序 + */ + PARAMETER_ERROR("40000100", "Bad request - Invalid query parameters :%s"), + BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION("40001300", "Error in SQL syntax parsing: %s"), + BAD_REQUEST_SQL_EXECUTION_EXCEPTION("40001301", "Error in database execution engine: %s"), + BAD_REQUEST_PARAM_SYNTAX_EXCEPTION("40001302", " Invalid Request Parameter: %s"), + + SCHEMA_AND_TABLE_NOT_CONSISTENCY("50001010", "Schema and table structure are not consistent: %s"), + HTTP_REQUEST_EXCEPTION("50001011", "Error in HTTP request : %s"), + SQL_REWRITE_AND_TRANSFORMATION_EXCEPTION("50001100", "Error in SQL rewrite and transformation: %s"), + SQL_EXECUTION_EXCEPTION("50001300", "Error in SQL execution: %s"), + SQL_FEDERATION_QUERY_EXCEPTION("50001500", "Error in SQL federation query: %s"), + + UNKNOWN_EXCEPTION("50000999", "An unknown error occurred: %s"); + + private String code; + private String message; + + CommonErrorCode(String code, String message) { + this.code = code; + this.message = message; + } +} diff --git a/src/main/java/com/mesalab/common/nacos/NacosConst.java b/src/main/java/com/mesalab/common/nacos/NacosConst.java index a4efb2e1..87bc919f 100644 --- a/src/main/java/com/mesalab/common/nacos/NacosConst.java +++ b/src/main/java/com/mesalab/common/nacos/NacosConst.java @@ -36,7 +36,7 @@ public class NacosConst { public static final String META_DATA_ID = "meta_data.json"; - public static final String SQL_DATASETS_VARIABLES = "sql_datasets_variables.json"; + public static final String DATASETS_VARIABLES = "datasets_variables.json"; public static final String CK_FILTER_DATA_ID = "ck-filter.json"; diff --git a/src/main/java/com/mesalab/common/utils/HazelcastInstanceMapUtil.java b/src/main/java/com/mesalab/common/utils/HazelcastInstanceMapUtil.java new file mode 100644 index 00000000..28c314ae --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/HazelcastInstanceMapUtil.java @@ -0,0 +1,49 @@ +package com.mesalab.common.utils; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONWriter; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.map.IMap; +import com.mesalab.qgw.model.basic.QueryCache; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +/** + * TODO + * + * @Classname HazelcastInstanceMapUtil + * @Date 2023/12/5 15:13 + * @Author wWei + */ +@Component +public class HazelcastInstanceMapUtil { + + private static HazelcastInstance hazelcastInstance; + private static final String AD_HOC_CACHE = "adHocCache"; + + @Autowired + public HazelcastInstanceMapUtil(@Qualifier("hazelcastInstance") HazelcastInstance hazelcastInstance) { + HazelcastInstanceMapUtil.hazelcastInstance = hazelcastInstance; + } + + public static IMap<String, String> retrieveMap() { + return hazelcastInstance.getMap(AD_HOC_CACHE); + } + + public static QueryCache put(String key, QueryCache value) { + String vStr = JSON.toJSONString(value, JSONWriter.Feature.WriteNulls, JSONWriter.Feature.LargeObject); + String old = retrieveMap().put(key, vStr); + return old == null ? null : JSON.parseObject(old, QueryCache.class); + } + + public static QueryCache get(String key) { + String v = retrieveMap().get(key); + return v == null ? null : JSON.parseObject(v, QueryCache.class); + } + + public static QueryCache remove(String key) { + String d = retrieveMap().remove(key); + return d == null ? null : JSON.parseObject(d, QueryCache.class); + } +} diff --git a/src/main/java/com/mesalab/common/utils/JsonMapper.java b/src/main/java/com/mesalab/common/utils/JsonMapper.java index 66b1c39d..896761bf 100644 --- a/src/main/java/com/mesalab/common/utils/JsonMapper.java +++ b/src/main/java/com/mesalab/common/utils/JsonMapper.java @@ -14,8 +14,6 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.util.JSONPObject; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.text.SimpleDateFormat; @@ -26,223 +24,224 @@ import java.util.TimeZone; /** * 简单封装Jackson，实现JSON String<->Java Object的Mapper. 封装不同的输出风格, * 使用不同的builder函数创建实例. - * + * * @author - * @version */ public class JsonMapper extends ObjectMapper { - private static final long serialVersionUID = 1L; - private static final Log log = LogFactory.get(); - - private static JsonMapper mapper; - - public JsonMapper() { - this(Include.ALWAYS); - } - - public JsonMapper(Include include) { - // 设置输出时包含属性的风格 - if (include != null) { - this.setSerializationInclusion(include); - } - // 允许单引号、允许不带引号的字段名称 - this.enableSimple(); - // 设置输入时忽略在JSON字符串中存在但Java对象实际没有的属性 - this.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - // 空值 - /* - * this.getSerializerProvider().setNullValueSerializer(new - * JsonSerializer<Object>(){ - * - * @Override public void serialize(Object value, JsonGenerator jgen, - * SerializerProvider provider) throws IOException, - * JsonProcessingException { jgen.writeString(""); } }); - */ - this.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); - // 进行HTML解码。 - this.registerModule(new SimpleModule().addSerializer(String.class, new JsonSerializer<String>() { - @Override - public void serialize(String value, JsonGenerator jgen, SerializerProvider provider) - throws IOException { - jgen.writeString(StringEscapeUtils.unescapeHtml4(value)); - } - })); - // 设置时区 - this.setTimeZone(TimeZone.getDefault());// getTimeZone("GMT+8:00") - } - - /** - * 创建只输出非Null且非Empty(如List.isEmpty)的属性到Json字符串的Mapper,建议在外部接口中使用. - */ - public static JsonMapper getInstance() { - if (mapper == null) { - mapper = new JsonMapper();// .enableSimple();只输出非Null且非Empty - } - return mapper; - } - - /** - * 创建只输出初始值被改变的属性到Json字符串的Mapper, 最节约的存储方式，建议在内部接口中使用。 - */ - public static JsonMapper nonDefaultMapper() { - if (mapper == null) { - mapper = new JsonMapper(Include.NON_DEFAULT); - } - return mapper; - } - - /** - * Object可以是POJO，也可以是Collection或数组。 如果对象为Null, 返回"null". 如果集合为空集合, 返回"[]". - */ - public String toJson(Object object) { - try { - return this.writeValueAsString(object); - } catch (IOException e) { - log.error("write to json string error:" , e); - return null; - } - } - - /** - * 反序列化POJO或简单Collection如List<String>. - * - * 如果JSON字符串为Null或"null"字符串, 返回Null. 如果JSON字符串为"[]", 返回空集合. - * - * 如需反序列化复杂Collection如List<MyBean>, 请使用fromJson(String,JavaType) - * - * @see #fromJson(String, JavaType) - */ - public <T> T fromJson(String jsonString, Class<T> clazz) { - if (StringUtils.isEmpty(jsonString)) { - return null; - } - try { - return this.readValue(jsonString, clazz); - } catch (IOException e) { - log.error("parse json string error:" , e); - return null; - } - } - - /** - * 反序列化复杂Collection如List<Bean>, 先使用函數createCollectionType构造类型,然后调用本函数. - * - * @see #createCollectionType(Class, Class...) - */ - @SuppressWarnings("unchecked") - public <T> T fromJson(String jsonString, JavaType javaType) { - if (StringUtils.isEmpty(jsonString)) { - return null; - } - try { - return (T) this.readValue(jsonString, javaType); - } catch (IOException e) { - log.error("parse json string error:" , e); - return null; - } - } - - /** - * 構造泛型的Collection Type如: ArrayList<MyBean>, - * 则调用constructCollectionType(ArrayList.class,MyBean.class) - * HashMap<String,MyBean>, 则调用(HashMap.class,String.class, MyBean.class) - */ - public JavaType createCollectionType(Class<?> collectionClass, Class<?>... elementClasses) { - return this.getTypeFactory().constructParametricType(collectionClass, elementClasses); - } - - /** - * 當JSON裡只含有Bean的部分屬性時，更新一個已存在Bean，只覆蓋該部分的屬性. - */ - @SuppressWarnings("unchecked") - public <T> T update(String jsonString, T object) { - try { - return (T) this.readerForUpdating(object).readValue(jsonString); - } catch (JsonProcessingException e) { - log.error("update json string:" + jsonString + " to object:" + object + " error.", e); - } catch (IOException e) { - log.error("update json string:" + jsonString + " to object:" + object + " error.", e); - } - return null; - } - - /** - * 輸出JSONP格式數據. - */ - public String toJsonP(String functionName, Object object) { - return toJson(new JSONPObject(functionName, object)); - } - - /** - * 設定是否使用Enum的toString函數來讀寫Enum, 為False時時使用Enum的name()函數來讀寫Enum, 默認為False. - * 注意本函數一定要在Mapper創建後, 所有的讀寫動作之前調用. - */ - public JsonMapper enableEnumUseToString() { - this.enable(SerializationFeature.WRITE_ENUMS_USING_TO_STRING); - this.enable(DeserializationFeature.READ_ENUMS_USING_TO_STRING); - return this; - } - - - /** - * 允许单引号 允许不带引号的字段名称 - */ - public JsonMapper enableSimple() { - this.configure(Feature.ALLOW_SINGLE_QUOTES, true); - this.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); - return this; - } - - /** - * 取出Mapper做进一步的设置或使用其他序列化API. - */ - public ObjectMapper getMapper() { - return this; - } - - /** - * 对象转换为JSON字符串 - * - * @param object - * @return - */ - public static String toJsonString(Object object) { - return JsonMapper.getInstance().toJson(object); - } - - /** - * JSON字符串转换为对象 - * - * @param jsonString - * @param clazz - * @return - */ - public static <T> T fromJsonString(String jsonString, Class<T> clazz) { - return JsonMapper.getInstance().fromJson(jsonString, clazz); - } - - /** - * JSON字符串转为集合对象 - * @param jsonString - * @param collectionClass - * @param elementClasses - * @return - */ - public static <T> T fromJsonString(String jsonString,Class<?> collectionClass, Class<?>... elementClasses) { - return JsonMapper.getInstance().fromJson(jsonString, JsonMapper.getInstance().createCollectionType(collectionClass,elementClasses)); - } - /** - * JSON字符串转换为对象 - * - * @param jsonString - * @param clazz - * @return - */ - public static Object fromJsonList(String jsonString, Class<?> clazz) { - JavaType javaType = JsonMapper.getInstance().createCollectionType(ArrayList.class, clazz); - return JsonMapper.getInstance().fromJson(jsonString,javaType ); - } + private static final long serialVersionUID = 1L; + private static final Log log = LogFactory.get(); + + private static JsonMapper mapper; + + public JsonMapper() { + this(Include.ALWAYS); + } + + public JsonMapper(Include include) { + // 设置输出时包含属性的风格 + if (include != null) { + this.setSerializationInclusion(include); + } + // 允许单引号、允许不带引号的字段名称 + this.enableSimple(); + // 设置输入时忽略在JSON字符串中存在但Java对象实际没有的属性 + this.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + // 空值 + /* + * this.getSerializerProvider().setNullValueSerializer(new + * JsonSerializer<Object>(){ + * + * @Override public void serialize(Object value, JsonGenerator jgen, + * SerializerProvider provider) throws IOException, + * JsonProcessingException { jgen.writeString(""); } }); + */ + this.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); + // 进行HTML解码。 + this.registerModule(new SimpleModule().addSerializer(String.class, new JsonSerializer<String>() { + @Override + public void serialize(String value, JsonGenerator jgen, SerializerProvider provider) + throws IOException { + jgen.writeString(StringEscapeUtils.unescapeHtml4(value)); + } + })); + // 设置时区 + this.setTimeZone(TimeZone.getDefault());// getTimeZone("GMT+8:00") + } + + /** + * 创建只输出非Null且非Empty(如List.isEmpty)的属性到Json字符串的Mapper,建议在外部接口中使用. + */ + public static JsonMapper getInstance() { + if (mapper == null) { + mapper = new JsonMapper();// .enableSimple();只输出非Null且非Empty + } + return mapper; + } + + /** + * 创建只输出初始值被改变的属性到Json字符串的Mapper, 最节约的存储方式，建议在内部接口中使用。 + */ + public static JsonMapper nonDefaultMapper() { + if (mapper == null) { + mapper = new JsonMapper(Include.NON_DEFAULT); + } + return mapper; + } + + /** + * Object可以是POJO，也可以是Collection或数组。 如果对象为Null, 返回"null". 如果集合为空集合, 返回"[]". + */ + public String toJson(Object object) { + try { + return this.writeValueAsString(object); + } catch (IOException e) { + log.error("write to json string error:", e); + return null; + } + } + + /** + * 反序列化POJO或简单Collection如List<String>. + * <p> + * 如果JSON字符串为Null或"null"字符串, 返回Null. 如果JSON字符串为"[]", 返回空集合. + * <p> + * 如需反序列化复杂Collection如List<MyBean>, 请使用fromJson(String,JavaType) + * + * @see #fromJson(String, JavaType) + */ + public <T> T fromJson(String jsonString, Class<T> clazz) { + if (StringUtils.isEmpty(jsonString)) { + return null; + } + try { + return this.readValue(jsonString, clazz); + } catch (IOException e) { + log.error("parse json string error:", e); + return null; + } + } + + /** + * 反序列化复杂Collection如List<Bean>, 先使用函數createCollectionType构造类型,然后调用本函数. + * + * @see #createCollectionType(Class, Class...) + */ + @SuppressWarnings("unchecked") + public <T> T fromJson(String jsonString, JavaType javaType) { + if (StringUtils.isEmpty(jsonString)) { + return null; + } + try { + return (T) this.readValue(jsonString, javaType); + } catch (IOException e) { + log.error("parse json string error:", e); + return null; + } + } + + /** + * 構造泛型的Collection Type如: ArrayList<MyBean>, + * 则调用constructCollectionType(ArrayList.class,MyBean.class) + * HashMap<String,MyBean>, 则调用(HashMap.class,String.class, MyBean.class) + */ + public JavaType createCollectionType(Class<?> collectionClass, Class<?>... elementClasses) { + return this.getTypeFactory().constructParametricType(collectionClass, elementClasses); + } + + /** + * 當JSON裡只含有Bean的部分屬性時，更新一個已存在Bean，只覆蓋該部分的屬性. + */ + @SuppressWarnings("unchecked") + public <T> T update(String jsonString, T object) { + try { + return (T) this.readerForUpdating(object).readValue(jsonString); + } catch (JsonProcessingException e) { + log.error("update json string:" + jsonString + " to object:" + object + " error.", e); + } catch (IOException e) { + log.error("update json string:" + jsonString + " to object:" + object + " error.", e); + } + return null; + } + + /** + * 輸出JSONP格式數據. + */ + public String toJsonP(String functionName, Object object) { + return toJson(new JSONPObject(functionName, object)); + } + + /** + * 設定是否使用Enum的toString函數來讀寫Enum, 為False時時使用Enum的name()函數來讀寫Enum, 默認為False. + * 注意本函數一定要在Mapper創建後, 所有的讀寫動作之前調用. + */ + public JsonMapper enableEnumUseToString() { + this.enable(SerializationFeature.WRITE_ENUMS_USING_TO_STRING); + this.enable(DeserializationFeature.READ_ENUMS_USING_TO_STRING); + return this; + } + + + /** + * 允许单引号 允许不带引号的字段名称 + */ + public JsonMapper enableSimple() { + this.configure(Feature.ALLOW_SINGLE_QUOTES, true); + this.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + return this; + } + + /** + * 取出Mapper做进一步的设置或使用其他序列化API. + */ + public ObjectMapper getMapper() { + return this; + } + + /** + * 对象转换为JSON字符串 + * + * @param object + * @return + */ + public static String toJsonString(Object object) { + return JsonMapper.getInstance().toJson(object); + } + + /** + * JSON字符串转换为对象 + * + * @param jsonString + * @param clazz + * @return + */ + public static <T> T fromJsonString(String jsonString, Class<T> clazz) { + return JsonMapper.getInstance().fromJson(jsonString, clazz); + } + + /** + * JSON字符串转为集合对象 + * + * @param jsonString + * @param collectionClass + * @param elementClasses + * @return + */ + public static <T> T fromJsonString(String jsonString, Class<?> collectionClass, Class<?>... elementClasses) { + return JsonMapper.getInstance().fromJson(jsonString, JsonMapper.getInstance().createCollectionType(collectionClass, elementClasses)); + } + + /** + * JSON字符串转换为对象 + * + * @param jsonString + * @param clazz + * @return + */ + public static Object fromJsonList(String jsonString, Class<?> clazz) { + JavaType javaType = JsonMapper.getInstance().createCollectionType(ArrayList.class, clazz); + return JsonMapper.getInstance().fromJson(jsonString, javaType); + } } diff --git a/src/main/java/com/mesalab/common/utils/JsonSchemaValidator.java b/src/main/java/com/mesalab/common/utils/JsonSchemaValidator.java new file mode 100644 index 00000000..911248b8 --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/JsonSchemaValidator.java @@ -0,0 +1,148 @@ +package com.mesalab.common.utils; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.io.resource.ClassPathResource; +import cn.hutool.core.io.resource.Resource; +import cn.hutool.core.util.StrUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.alibaba.fastjson2.JSON; +import com.fasterxml.jackson.databind.JsonNode; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.networknt.schema.*; +import lombok.Data; +import org.apache.commons.io.IOUtils; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.*; + + +public class JsonSchemaValidator { + private static final Log log = LogFactory.get(); + private static final JsonSchemaValidator INSTANCE = new JsonSchemaValidator(); + private final List<ValidationRule> rules; + private static final String ENUM_VALUES = "\"$ENUM_VALUES$\""; + private static final String EXPRESSION_FIELDS = "expression_fields"; + private final JsonSchemaFactory schemaFactory; + private final SchemaValidatorsConfig schemaValidatorsConfig; + + private JsonSchemaValidator() { + this.schemaFactory = JsonSchemaFactory + .builder(com.networknt.schema.JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7)) + .build(); + this.schemaValidatorsConfig = new SchemaValidatorsConfig(); + this.schemaValidatorsConfig.setLocale(Locale.ENGLISH); + this.rules = new ArrayList<>(); + } + + public static JsonSchemaValidator getInstance() { + return INSTANCE; + } + + public JsonSchemaValidator addRule(String jsonSchemaFileName, String type) { + if (StrUtil.isNotBlank(jsonSchemaFileName) && StrUtil.isNotBlank(type)) { + ValidationRule rule = new ValidationRule(jsonSchemaFileName, type); + if (!rules.contains(rule)) { + rules.add(rule); + } + return this; + } else { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + "Json-schema ruleName and type cannot be empty."); + } + } + + + public boolean validateSchema(String jsonStr) { + for (ValidationRule rule : rules) { + if (rule.getType().equals("schema")){ + Map map = JSON.parseObject(jsonStr, Map.class); + String tableName = String.valueOf(map.get("name")); + String jsonString = getJsonSchema(rule.getJsonSchemaFileName(), map); + JsonSchema schema = schemaFactory.getSchema(jsonString, schemaValidatorsConfig); + Set<ValidationMessage> validate = schema.validate(jsonStr, InputFormat.JSON); + if (!CollectionUtil.isEmpty(validate)) { + Iterator<ValidationMessage> iterator = validate.iterator(); + while (iterator.hasNext()) { + ValidationMessage next = iterator.next(); + String message = next.getMessage(); + log.error(String.format("%s schema validation fail: %s", tableName, message)); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format("%s schema validation fail: %s", tableName, message)); + } + } + } + } + return true; + } + + public boolean validateDSL(String jsonStr) { + for (ValidationRule rule : rules) { + if (rule.getType().equals("dsl")){ + try { + Resource resource = new ClassPathResource(rule.getJsonSchemaFileName()); + String jsonSchemaStr = new String(IOUtils.toByteArray(resource.getStream()), StandardCharsets.UTF_8); + JsonSchema schema = schemaFactory.getSchema(jsonSchemaStr, schemaValidatorsConfig); + Set<ValidationMessage> validate = schema.validate(jsonStr, InputFormat.JSON); + if (!CollectionUtil.isEmpty(validate)) { + Iterator<ValidationMessage> iterator = validate.iterator(); + while (iterator.hasNext()) { + ValidationMessage next = iterator.next(); + String message = next.getMessage(); + log.error(String.format("DSL validation fail: %s", message)); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format("DSL validation fail: %s", message)); + } + } + } catch (IOException e) { + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), e.getMessage()); + } + } + } + return true; + } + + private static String getJsonSchema(String jsonSchemaFileName, Map map) { + try { + Resource resource = new ClassPathResource(jsonSchemaFileName); + String jsonSchemaStr = new String(IOUtils.toByteArray(resource.getStream()), StandardCharsets.UTF_8); + StringBuffer sb = new StringBuffer(); + if (jsonSchemaStr.contains(ENUM_VALUES)) { + List<Map<String, Object>> fields = (List<Map<String, Object>>) map.get("fields"); + for (Map<String, Object> field : fields) { + String name = String.valueOf(field.get("name")); + sb.append("\"").append(name).append("\"").append(","); + } + if (map.containsKey("doc")) { + Map docMap = (Map) map.get("doc"); + if (docMap.containsKey(EXPRESSION_FIELDS)) { + List<Map<String, Object>> expressionFields = (List<Map<String, Object>>) docMap.get(EXPRESSION_FIELDS); + for (Map<String, Object> field : expressionFields) { + String name = String.valueOf(field.get("name")); + sb.append("\"").append(name).append("\"").append(","); + } + } + } + sb.setLength(sb.length() - 1); + return jsonSchemaStr.replace(ENUM_VALUES, sb.toString()); + } else { + return jsonSchemaStr; + } + } catch (IOException e) { + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), e.getMessage()); + } + } + + @Data + public static class ValidationRule { + private final String jsonSchemaFileName; + private final String type; + } +} diff --git a/src/main/java/com/mesalab/common/utils/MDCUtil.java b/src/main/java/com/mesalab/common/utils/MDCUtil.java new file mode 100644 index 00000000..88edd2c8 --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/MDCUtil.java @@ -0,0 +1,23 @@ +package com.mesalab.common.utils; + +import org.slf4j.MDC; + +/** + * TODO + * + * @Classname MDCUtil + * @Date 2024/1/10 18:11 + * @Author wWei + */ +public class MDCUtil { + + public static final String TRACE_ID = "traceId"; + + public static String getTraceId() { + return MDC.get(TRACE_ID); + } + + public static String generateTraceId() { + return RandomUtil.getUUID(); + } +} diff --git a/src/main/java/com/mesalab/common/utils/QueryCacheUtils.java b/src/main/java/com/mesalab/common/utils/QueryCacheUtils.java index 92a95f1a..8385596a 100644 --- a/src/main/java/com/mesalab/common/utils/QueryCacheUtils.java +++ b/src/main/java/com/mesalab/common/utils/QueryCacheUtils.java @@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory; import java.util.List; import java.util.concurrent.TimeUnit; +@Deprecated public class QueryCacheUtils { private static Cache<String,Object> cache; diff --git a/src/main/java/com/mesalab/common/utils/RandomNumberGenerator.java b/src/main/java/com/mesalab/common/utils/RandomNumberGenerator.java new file mode 100644 index 00000000..784e3dd4 --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/RandomNumberGenerator.java @@ -0,0 +1,80 @@ +package com.mesalab.common.utils; + +import java.text.DecimalFormat; +import java.util.Random; + +/** + * TODO + * + * @Classname RandomNumberGenerator + * @Date 2024/3/7 18:16 + * @Author wWei + */ + +public class RandomNumberGenerator { + private static final double MIN_PERCENTAGE_CHANGE = 0.01; + private static final double MAX_PERCENTAGE_CHANGE = 0.1; + private static final int MIN_INIT_VALUE = 1000; + private static final int MAX_INIT_VALUE = 1000000; + + /** + * Desc: Randomly generate an double within the range determined by the initial value and a percentage change ranging from 1% to 10%. + * + * @param initialValue initial value + * @param isDesc true increment, false decrement, null random + * @return {@link double} + * @created by wWei + * @date 2024/3/7 18:22 + */ + public static double generateRelativeRandomDouble(Double initialValue, Boolean isDesc) { + Random random = new Random(); + if (initialValue == null) { + initialValue = MIN_INIT_VALUE + (MAX_INIT_VALUE - MIN_INIT_VALUE) * random.nextDouble(); + } + DecimalFormat df = new DecimalFormat("#.##"); + + if (isDesc == null) { + isDesc = random.nextBoolean(); + } + double percentageChange = MIN_PERCENTAGE_CHANGE + random.nextDouble() * (MAX_PERCENTAGE_CHANGE - MIN_PERCENTAGE_CHANGE); + double v = !isDesc ? initialValue * (1 + percentageChange) : initialValue * (1 - percentageChange); + return Double.parseDouble(df.format(v)); + } + + /** + * Desc: Randomly generate an integer within the range determined by the initial value and a percentage change ranging from 1% to 10%. + * + * @param initialValue initial value + * @param isDesc true increment, false decrement, null random + * @return {@link double} + * @created by wWei + * @date 2024/3/7 18:22 + */ + public static int generateRelativeRandomInt(Integer initialValue, Boolean isDesc) { + Random random = new Random(); + if (initialValue == null) { + initialValue = random.nextInt(MAX_INIT_VALUE - MIN_INIT_VALUE + 1) + MIN_INIT_VALUE; + } + + if (isDesc == null) { + isDesc = random.nextBoolean(); + } + double percentageChange = MIN_PERCENTAGE_CHANGE + random.nextDouble() * (MAX_PERCENTAGE_CHANGE - MIN_PERCENTAGE_CHANGE); + double v = !isDesc ? initialValue * (1 + percentageChange) : initialValue * (1 - percentageChange); + return (int) v; + } + + /** + * Desc: Randomly generate an integer within the range of [min, max] + * + * @param min + * @param max + * @return {@link int} + * @created by wWei + * @date 2024/3/11 15:49 + */ + public static int generateRangeRandomInt(int min, int max) { + return new Random().nextInt(max - min + 1) + min; + } + +} diff --git a/src/main/java/com/mesalab/common/utils/SavedQueryResultUtils.java b/src/main/java/com/mesalab/common/utils/SavedQueryResultUtils.java index 195fc605..b484b4c2 100644 --- a/src/main/java/com/mesalab/common/utils/SavedQueryResultUtils.java +++ b/src/main/java/com/mesalab/common/utils/SavedQueryResultUtils.java @@ -5,13 +5,12 @@ import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.qgw.model.basic.QueryProfile; +import com.mesalab.qgw.model.basic.SQLQueryContext; import com.mesalab.qgw.model.basic.HBaseAPISource; -import com.mesalab.qgw.service.impl.QueryServiceImpl; +import com.mesalab.qgw.service.SQLSyncQueryService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import java.util.List; import java.util.Map; @@ -23,16 +22,9 @@ import java.util.Map; @Component public class SavedQueryResultUtils { private static final Log log = LogFactory.get(); - @Autowired - private HBaseAPISource hBaseAPISourceCache; private static HBaseAPISource hBaseAPISource; - - @PostConstruct - public void init() { - hBaseAPISource = this.hBaseAPISourceCache; - } - + private static SQLSyncQueryService sqlSyncQueryService; public static BaseResult<Object> getByRowKey(List<String> rowKeys) { List<Map<String, Object>> result = query(rowKeys); @@ -41,14 +33,23 @@ public class SavedQueryResultUtils { } private static List<Map<String, Object>> query(List<String> rowKeys) { - QueryServiceImpl queryService = new QueryServiceImpl(); - QueryProfile param = new QueryProfile(); + SQLQueryContext param = new SQLQueryContext(); StringBuffer sb = new StringBuffer(); sb.append("select ROWKEY, ").append(hBaseAPISource.getColumnName()); sb.append(" from ").append(hBaseAPISource.getDbName()).append(".").append(hBaseAPISource.getTableName()); sb.append(" where ").append("ROWKEY IN ('").append(StrUtil.join("', '", rowKeys)).append("')"); - param.setQuery(sb.toString()); - BaseResult baseResult = queryService.executeQuery(param); + param.setOriginalSQL(sb.toString()); + BaseResult baseResult = sqlSyncQueryService.executeQuery(param); return (List<Map<String, Object>>) baseResult.getData(); } + + @Autowired + public void setJobExecuteService(HBaseAPISource hBaseAPISource) { + SavedQueryResultUtils.hBaseAPISource = hBaseAPISource; + } + + @Autowired + public void sqlSyncQueryService(SQLSyncQueryService sqlSyncQueryService) { + SavedQueryResultUtils.sqlSyncQueryService = sqlSyncQueryService; + } } diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/AutoPeriodHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/AutoPeriodHelper.java new file mode 100644 index 00000000..0ce62ff6 --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/sqlparser/AutoPeriodHelper.java @@ -0,0 +1,469 @@ +package com.mesalab.common.utils.sqlparser; + +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUnit; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.alibaba.fastjson2.JSON; +import com.google.common.collect.Maps; +import com.mesalab.common.utils.SpringContextUtil; +import com.mesalab.qgw.constant.DslIdentifierNameConst; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import com.mesalab.qgw.service.DatabaseService; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.ExplainStatement; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.*; +import net.sf.jsqlparser.util.TablesNamesFinder; +import org.joda.time.Period; + +import java.util.*; + +public class AutoPeriodHelper { + private static final Log log = LogFactory.get(); + private static final DatabaseService databaseService = (DatabaseService) SpringContextUtil.getBean("databaseService"); + private static final String TIME_FLOOR_WITH_FILL = "TIME_FLOOR_WITH_FILL"; + private static final String RATE = "RATE"; + private static final String CHART_GRANULARITY = "CHART_GRANULARITY"; + private static final String SAMPLE_GRANULARITY = "SAMPLE_GRANULARITY"; + private static final String SCHEMA_DOC_MEASUREMENTS_KEY = "measurements"; + private static final String SKIP_COMPACTION_OFFSET_FROM_LATEST = "skip_compaction_offset_from_latest"; + private static final String COMPACTION_PERIOD = "compaction_period"; + private static final String INGESTION_PERIOD = "ingestion_period"; + private static final String PT0S = "PT0S"; + private static final String PT1S = "PT1S"; + private static final String P1D = "P1D"; + private static final Map<Long, String> STANDARD_CHART_PERIOD_MAP = new TreeMap<>(); + private static final Map<Long, String> STANDARD_SAMPLE_PERIOD_MAP = new TreeMap<>(); + + static { + STANDARD_CHART_PERIOD_MAP.put(300L, "PT1S"); + STANDARD_CHART_PERIOD_MAP.put(3600L, "PT30S"); + STANDARD_CHART_PERIOD_MAP.put(3600 * 12L, "PT1M"); + STANDARD_CHART_PERIOD_MAP.put(86400 * 3L, "PT5M"); + STANDARD_CHART_PERIOD_MAP.put(86400 * 30L, "PT1H"); + STANDARD_CHART_PERIOD_MAP.put(Long.MAX_VALUE, "P1D"); + STANDARD_SAMPLE_PERIOD_MAP.put(3600L, "PT1S"); + STANDARD_SAMPLE_PERIOD_MAP.put(3600 * 12L, "PT30S"); + STANDARD_SAMPLE_PERIOD_MAP.put(86400 * 3L, "PT1M"); + STANDARD_SAMPLE_PERIOD_MAP.put(86400 * 30L, "PT5M"); + STANDARD_SAMPLE_PERIOD_MAP.put(Long.MAX_VALUE, "PT1H"); + } + + private AutoPeriodHelper() { + } + + public static Statement buildSqlGranularity(Statement statement) { + DateTime now = new DateTime(); + if (statement instanceof ExplainStatement) { + ExplainStatement explainStatement = (ExplainStatement) statement; + buildSelectStatement(now, explainStatement.getStatement()); + return explainStatement; + } else if (statement instanceof Select) { + buildSelectStatement(now, (Select) statement); + return statement; + } + return statement; + } + + public static void buildDslGranularity(DSLQueryRequestParam dslQueryRequestParam) { + Map<String, Map<String, Object>> measurements = getMeasurementsMap(dslQueryRequestParam); + if (measurements.isEmpty()) { + return; + } + String granularity = dslQueryRequestParam.getGranularity(); + if (StrUtil.isEmpty(granularity)) { + List<String> intervals = dslQueryRequestParam.getIntervals(); + if (intervals != null && intervals.size() == 1) { + String[] split = intervals.get(0).split("/"); + DateTime start = DateUtil.parse(split[0]); + DateTime end = DateUtil.parse(split[1]); + dslQueryRequestParam.setGranularity(getGranularityPeriod(new DateTime(), start, end, measurements, true)); + } + } else { + if (granularity.startsWith(CHART_GRANULARITY)) { + dslQueryRequestParam.setGranularity(parseGranularity(granularity, measurements, dslQueryRequestParam, true)); + } else if (granularity.startsWith(SAMPLE_GRANULARITY)) { + parseGranularity(granularity, measurements, dslQueryRequestParam, false); + } + } + } + + private static void buildSelectStatement(DateTime now, Select select) { + Map<String, Map<String, Object>> measurements = Maps.newHashMap(); + List<String> tableList = SQLHelper.getTableName(select); + for (String table : tableList) { + String measurement = databaseService.getValueByKeyInSchemaDoc(table, SCHEMA_DOC_MEASUREMENTS_KEY); + if (StrUtil.isNotEmpty(measurement)) { + measurements.put(table, JSON.parseObject(measurement, Map.class)); + } + } + select.getSelectBody().accept(new SelectAdapter(now, measurements)); + } + + private static String parseGranularity(String granularity, Map<String, Map<String, Object>> measurements, DSLQueryRequestParam dslQueryRequestParam, boolean isChartGranularity) { + try { + Expression expression = CCJSqlParserUtil.parseExpression(granularity); + if (expression instanceof Function) { + Function fun = (Function) expression; + Expression expression1 = fun.getParameters().getExpressions().get(0); + Expression expression2 = fun.getParameters().getExpressions().get(1); + DateTime start = DateUtil.parse(((StringValue) expression1).getValue()); + DateTime end = DateUtil.parse(((StringValue) expression2).getValue()); + return getGranularityPeriod(new DateTime(), start, end, measurements, isChartGranularity); + } + } catch (JSQLParserException e) { + log.error("Parse DSL auto granularity error", e); + } + return dslQueryRequestParam.getGranularity(); + } + + private static Map<String, Map<String, Object>> getMeasurementsMap(DSLQueryRequestParam dslQueryRequestParam) { + if (DslIdentifierNameConst.IP_LEARNING_FQDN_RELATE_IP.equalsIgnoreCase(dslQueryRequestParam.getName()) + || DslIdentifierNameConst.IP_LEARNING_ACTIVE_IP.equalsIgnoreCase(dslQueryRequestParam.getName())) { + return Maps.newHashMap(); + } + Map<String, Map<String, Object>> measurements = Maps.newHashMap(); + if (StrUtil.isEmpty(dslQueryRequestParam.getDataSource())) { + Set<String> tables = DslIdentifierNameConst.IDENTIFIER_NAME_SOURCE_MAPPING.get(dslQueryRequestParam.getName()); + if (tables != null && !tables.isEmpty()) { + for (String table : tables) { + String measurement = databaseService.getValueByKeyInSchemaDoc(table, SCHEMA_DOC_MEASUREMENTS_KEY); + if (StrUtil.isNotEmpty(measurement)) { + measurements.put(table, JSON.parseObject(measurement, Map.class)); + } + } + } + } else { + String measurement = databaseService.getValueByKeyInSchemaDoc(dslQueryRequestParam.getDataSource(), SCHEMA_DOC_MEASUREMENTS_KEY); + if (StrUtil.isNotEmpty(measurement)) { + measurements.put(dslQueryRequestParam.getDataSource(), JSON.parseObject(measurement, Map.class)); + } + } + return measurements; + } + + private static String getGranularityPeriod(DateTime now, DateTime startTime, DateTime endTime, Map<String, Map<String, Object>> measurements, boolean isChartGranularity) { + Map<String, String> measurement = getMeasurementWithEffectiveGranularity(measurements); + long duration = DateUtil.between(startTime, endTime, DateUnit.SECOND, false); + DateTime agoTime = DateUtil.offsetSecond(now, -Period.parse(measurement.get(SKIP_COMPACTION_OFFSET_FROM_LATEST)).toStandardSeconds().getSeconds()); + String dataPeriod = getDataPeriod(measurement, agoTime, startTime); + int dataPeriodSeconds = Period.parse(dataPeriod).toStandardSeconds().getSeconds(); + return getStandardPeriod(isChartGranularity, duration, dataPeriodSeconds); + } + + private static Map<String, String> getMeasurementWithEffectiveGranularity(Map<String, Map<String, Object>> measurements) { + Map<String, String> measurement = Maps.newHashMap(); + measurements.values().forEach(m -> { + m.forEach((k, v) -> { + if (!SKIP_COMPACTION_OFFSET_FROM_LATEST.equalsIgnoreCase(k) + && !COMPACTION_PERIOD.equalsIgnoreCase(k) + && !INGESTION_PERIOD.equalsIgnoreCase(k)) { + return; + } + if (measurement.containsKey(k)) { + String oldPeriod = measurement.get(k); + if (SKIP_COMPACTION_OFFSET_FROM_LATEST.equalsIgnoreCase(k)) { + if (Period.parse(v.toString()).toStandardSeconds().getSeconds() < Period.parse(oldPeriod).toStandardSeconds().getSeconds()) { + measurement.put(k, v.toString()); + } + } else { + if (Period.parse(v.toString()).toStandardSeconds().getSeconds() > Period.parse(oldPeriod).toStandardSeconds().getSeconds()) { + measurement.put(k, v.toString()); + } + } + } else { + measurement.put(k, v.toString()); + } + }); + }); + setDefaultValues(measurement); + return measurement; + } + + private static void setDefaultValues(Map<String, String> measurement) { + String compactionPeriod = measurement.getOrDefault(COMPACTION_PERIOD, null); + String ingestionPeriod = measurement.getOrDefault(INGESTION_PERIOD, null); + measurement.putIfAbsent(SKIP_COMPACTION_OFFSET_FROM_LATEST, PT0S); + + if (compactionPeriod == null && ingestionPeriod == null) { + measurement.put(COMPACTION_PERIOD, PT1S); + measurement.put(INGESTION_PERIOD, PT1S); + } else if (compactionPeriod == null) { + measurement.put(COMPACTION_PERIOD, ingestionPeriod); + } else if (ingestionPeriod == null) { + measurement.put(INGESTION_PERIOD, compactionPeriod); + } + + } + + private static String getDataPeriod(Map<String, String> measurement, DateTime splitTime, DateTime startTime) { + String dataPeriod = null; + if (DateUtil.compare(splitTime, startTime) > 0) { + dataPeriod = measurement.get(COMPACTION_PERIOD); + } + if (StrUtil.isEmpty(dataPeriod)) { + dataPeriod = measurement.get(INGESTION_PERIOD); + } + return dataPeriod; + } + + private static String getStandardPeriod(boolean isChartGranularity, long duration, int dataPeriodSeconds) { + Map<Long, String> standard = isChartGranularity ? STANDARD_CHART_PERIOD_MAP : STANDARD_SAMPLE_PERIOD_MAP; + for (Map.Entry<Long, String> entry : standard.entrySet()) { + if (duration <= entry.getKey() && dataPeriodSeconds <= Period.parse(entry.getValue()).toStandardSeconds().getSeconds()) { + return entry.getValue(); + } + } + return P1D; + } + + private static class SelectAdapter extends SelectVisitorAdapter { + private final DateTime now; + private final Map<String, Map<String, Object>> measurements; + + public SelectAdapter(DateTime now, Map<String, Map<String, Object>> measurements) { + this.now = now; + this.measurements = measurements; + } + + @Override + public void visit(PlainSelect plainSelect) { + List<SelectItem> selectItems = plainSelect.getSelectItems(); + if (selectItems != null) { + for (SelectItem selectItem : selectItems) { + SelectItemAdapter selectItemAdapter = new SelectItemAdapter(now, measurements); + selectItem.accept(selectItemAdapter); + } + } + + FromItem fromItem = plainSelect.getFromItem(); + if (fromItem != null) { + fromItem.accept(new FromItemAdapter(now, measurements)); + } + + Expression where = plainSelect.getWhere(); + if (where != null) { + where.accept(new ExpressionAdapter(now, measurements)); + } + + GroupByElement groupBy = plainSelect.getGroupBy(); + if (groupBy != null && groupBy.getGroupByExpressionList() != null && !groupBy.getGroupByExpressionList().getExpressions().isEmpty()) { + groupBy.accept(new GroupByAdapter(now, measurements)); + } + + Expression having = plainSelect.getHaving(); + if (having != null) { + having.accept(new ExpressionAdapter(now, measurements)); + } + + List<OrderByElement> orderByElements = plainSelect.getOrderByElements(); + if (orderByElements != null) { + for (OrderByElement orderByElement : orderByElements) { + orderByElement.accept(new OrderByAdapter(now, measurements)); + } + } + } + + @Override + public void visit(SetOperationList setOpList) { + for (SelectBody select : setOpList.getSelects()) { + if (select instanceof PlainSelect) { + select.accept(this); + } + } + } + } + + private static class SelectItemAdapter extends SelectItemVisitorAdapter { + + private final DateTime now; + private final Map<String, Map<String, Object>> measurements; + + public SelectItemAdapter(DateTime now, Map<String, Map<String, Object>> measurements) { + this.now = now; + this.measurements = measurements; + } + + @Override + public void visit(SelectExpressionItem selectExpressionItem) { + Expression expression = selectExpressionItem.getExpression(); + FunctionAdapter functionAdapter = new FunctionAdapter(now, measurements); + expression.accept(functionAdapter); + } + + } + + private static class FromItemAdapter extends FromItemVisitorAdapter { + private final DateTime now; + private final Map<String, Map<String, Object>> measurements; + + public FromItemAdapter(DateTime now, Map<String, Map<String, Object>> measurements) { + this.now = now; + this.measurements = measurements; + } + + @Override + public void visit(SubSelect subSelect) { + SelectBody selectBody = subSelect.getSelectBody(); + selectBody.accept(new SelectAdapter(now, measurements)); + } + } + + private static class ExpressionAdapter extends ExpressionVisitorAdapter { + private final DateTime now; + private final Map<String, Map<String, Object>> measurements; + + public ExpressionAdapter(DateTime now, Map<String, Map<String, Object>> measurements) { + this.now = now; + this.measurements = measurements; + } + + @Override + public void visit(SubSelect subSelect) { + SelectBody selectBody = subSelect.getSelectBody(); + selectBody.accept(new SelectAdapter(now, measurements)); + } + + @Override + public void visit(Function function) { + FunctionAdapter functionAdapter = new FunctionAdapter(now, measurements); + function.accept(functionAdapter); + } + } + + private static class GroupByAdapter implements GroupByVisitor { + private final DateTime now; + private final Map<String, Map<String, Object>> measurements; + + public GroupByAdapter(DateTime now, Map<String, Map<String, Object>> measurements) { + this.now = now; + this.measurements = measurements; + } + + @Override + public void visit(GroupByElement groupByElement) { + if (groupByElement.getGroupByExpressionList() == null + || groupByElement.getGroupByExpressionList().getExpressions() == null) { + return; + } + for (Expression expression : groupByElement.getGroupByExpressionList().getExpressions()) { + if (!(expression instanceof Function)) { + continue; + } + FunctionAdapter functionAdapter = new FunctionAdapter(now, measurements); + expression.accept(functionAdapter); + } + } + } + + private static class OrderByAdapter implements OrderByVisitor { + private final DateTime now; + private final Map<String, Map<String, Object>> measurements; + + public OrderByAdapter(DateTime now, Map<String, Map<String, Object>> measurements) { + this.now = now; + this.measurements = measurements; + } + + @Override + public void visit(OrderByElement orderBy) { + if (orderBy.getExpression() == null) { + return; + } + if (!(orderBy.getExpression() instanceof Function)) { + return; + } + FunctionAdapter functionAdapter = new FunctionAdapter(now, measurements); + orderBy.getExpression().accept(functionAdapter); + + } + } + + private static class FunctionAdapter extends ExpressionVisitorAdapter { + + private final DateTime now; + private final Map<String, Map<String, Object>> measurements; + + public FunctionAdapter(DateTime now, Map<String, Map<String, Object>> measurements) { + this.now = now; + this.measurements = measurements; + } + + @Override + public void visit(Function function) { + if (isTimeFloorWithFill(function)) { + buildNewFunction(function); + } else if (isRate(function)) { + buildNewFunction(function); + } else { + Optional.ofNullable(function.getParameters()).ifPresent(o -> { + for (Expression expression : o.getExpressions()) { + expression.accept(this); + } + }); + } + } + + private void buildNewFunction(Function function) { + Expression period = function.getParameters().getExpressions().get(1); + if (isChartGranularity(period)) { + Function periodFunction = (Function) period; + DateTime startTime = DateUtil.parse(((StringValue) periodFunction.getParameters().getExpressions().get(0)).getValue()); + DateTime endTime = DateUtil.parse(((StringValue) periodFunction.getParameters().getExpressions().get(1)).getValue()); + String chartGranularityPeriod = getGranularityPeriod(now, startTime, endTime, measurements, true); + function.getParameters().getExpressions().set(1, new StringValue("'" + chartGranularityPeriod + "'")); + } else if (isSampleGranularity(period)) { + Function periodFunction = (Function) period; + DateTime startTime = DateUtil.parse(((StringValue) periodFunction.getParameters().getExpressions().get(0)).getValue()); + DateTime endTime = DateUtil.parse(((StringValue) periodFunction.getParameters().getExpressions().get(1)).getValue()); + String chartGranularityPeriod = getGranularityPeriod(now, startTime, endTime, measurements, false); + function.getParameters().getExpressions().set(1, new StringValue("'" + chartGranularityPeriod + "'")); + } + } + } + + private static boolean isChartGranularity(Expression expression) { + if (!(expression instanceof Function)) { + return false; + } + Function function = (Function) expression; + return CHART_GRANULARITY.equalsIgnoreCase(function.getName()) + && function.getParameters().getExpressions().size() == 2; + } + + private static boolean isSampleGranularity(Expression expression) { + if (!(expression instanceof Function)) { + return false; + } + Function function = (Function) expression; + return SAMPLE_GRANULARITY.equalsIgnoreCase(function.getName()) + && function.getParameters().getExpressions().size() == 2; + } + + private static boolean isTimeFloorWithFill(Expression expression) { + if (!(expression instanceof Function)) { + return false; + } + Function function = (Function) expression; + return TIME_FLOOR_WITH_FILL.equalsIgnoreCase(function.getName()) + && function.getParameters().getExpressions().size() >= 2; + } + + private static boolean isRate(Expression expression) { + if (!(expression instanceof Function)) { + return false; + } + Function function = (Function) expression; + return RATE.equalsIgnoreCase(function.getName()) + && function.getParameters().getExpressions().size() >= 2; + } + +} diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/ColumnCategoryHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/ColumnCategoryHelper.java new file mode 100644 index 00000000..61705022 --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/sqlparser/ColumnCategoryHelper.java @@ -0,0 +1,96 @@ +package com.mesalab.common.utils.sqlparser; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.geedgenetworks.utils.StringUtil; +import com.jayway.jsonpath.JsonPath; +import com.mesalab.common.enums.DBEngineType; +import com.mesalab.qgw.constant.MetaConst; +import com.mesalab.qgw.dialect.ClickHouseDialect; +import com.mesalab.qgw.dialect.DruidDialect; +import com.mesalab.qgw.model.basic.SelectStatement; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static com.mesalab.qgw.constant.MetaConst.*; + +/** + * TODO + * + * @Classname ColumnCategoryHelper + * @Date 2024/3/7 15:52 + * @Author wWei + */ +public class ColumnCategoryHelper { + private static final Log log = LogFactory.get(); + + private ColumnCategoryHelper() { + } + + public static Object expandMetaCategory(Object columnMeta, SelectStatement selectStatement, String dbEngine) { + if (StringUtil.isEmpty(selectStatement) || StringUtil.isEmpty(selectStatement.getExecSQL()) || StringUtil.isEmpty(columnMeta)) { + return columnMeta; + } + List<Map<String, String>> result = JsonPath.read(columnMeta, "$"); + Map<String, String> groupDimension = selectStatement.getGroupDimension(); + Map<String, String> aliasFields = selectStatement.getAliasFields(); + meta: + for (Map<String, String> map : result) { + if (StrUtil.isEmptyIfStr(groupDimension)) { + map.put(META_CATEGORY, META_CATEGORY_METRIC); + continue; + } + String field = String.valueOf(map.get(META_NAME)); + for (String dimensionKey : groupDimension.keySet()) { + if (Objects.equals(field, dimensionKey) || Objects.equals(field, groupDimension.get(dimensionKey))) { + map.put(META_CATEGORY, META_CATEGORY_DIMENSION); + continue meta; + } + try { + String selectItemExpr = aliasFields.get(field) == null ? SQLFunctionUtil.generateDateFunction(field, dbEngine) : aliasFields.get(field); + String dimensionExpr = groupDimension.get(dimensionKey) == null ? SQLFunctionUtil.generateDateFunction(dimensionKey, dbEngine) : groupDimension.get(dimensionKey); + if (dimensionExpr != null && selectItemExpr != null) { + ExpressionColumnCollectAdapter selectItemAdapter = new ExpressionColumnCollectAdapter(); + Expression expr1 = CCJSqlParserUtil.parseExpression(selectItemExpr); + if (SQLFunctionUtil.pAggregateFunStandard.matcher(expr1.toString()).find()) { + map.put(META_CATEGORY, META_CATEGORY_METRIC); + continue meta; + } + if (DBEngineType.CLICKHOUSE.getValue().equalsIgnoreCase(dbEngine) && ClickHouseDialect.pAggregateFunSpecificOfCK.matcher(expr1.toString()).find()) { + map.put(META_CATEGORY, META_CATEGORY_METRIC); + continue meta; + } + if (DBEngineType.DRUID.getValue().equalsIgnoreCase(dbEngine) && DruidDialect.pAggregateFunSpecificOfDruid.matcher(expr1.toString()).find()) { + map.put(META_CATEGORY, META_CATEGORY_METRIC); + continue meta; + } + expr1.accept(selectItemAdapter); + Set<String> selectItemColumn = selectItemAdapter.getColumns(); + + ExpressionColumnCollectAdapter dimensionAdapter = new ExpressionColumnCollectAdapter(); + Expression expr2 = CCJSqlParserUtil.parseExpression(dimensionExpr); + expr2.accept(dimensionAdapter); + Set<String> dimensionColumn = dimensionAdapter.getColumns(); + + boolean hasDuplicates = selectItemColumn.stream().anyMatch(dimensionColumn::contains); + if (hasDuplicates) { + map.put(META_CATEGORY, META_CATEGORY_DIMENSION); + continue meta; + } + } + } catch (JSQLParserException e) { + log.warn("build meta error, message is: {}", e.getMessage()); + } + } + map.put(META_CATEGORY, META_CATEGORY_METRIC); + } + return result; + } +} diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/CondExpressionHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/CondExpressionHelper.java index bb93ba8b..5f821b59 100644 --- a/src/main/java/com/mesalab/common/utils/sqlparser/CondExpressionHelper.java +++ b/src/main/java/com/mesalab/common/utils/sqlparser/CondExpressionHelper.java @@ -60,23 +60,6 @@ public class CondExpressionHelper { return orderBy.getList(); } - /** - * Desc: Get filed name - * - * @param column - * @return {@link String} - * @created by wWei - * @date 2023/3/2 09:58 - */ - public static String getFieldName(Column column) { - String columnName = column.getColumnName(); - if ((columnName.startsWith("\"") && columnName.endsWith("\"")) - || (columnName.startsWith("`") && columnName.endsWith("`"))) { - columnName = columnName.substring(1, columnName.length() - 1); - } - return columnName; - } - private static class ExprFieldParser extends ExpressionVisitorAdapter { private final List<String> list = Lists.newArrayList(); @@ -86,7 +69,7 @@ public class CondExpressionHelper { @Override public void visit(Column column) { - list.add(getFieldName(column)); + list.add(SQLHelper.removeQuotesAndBackticks(column.getColumnName())); } } @@ -109,7 +92,7 @@ public class CondExpressionHelper { } return; } - String columnName = getFieldName(column); + String columnName = SQLHelper.removeQuotesAndBackticks(column.getColumnName()); if (map.containsKey(columnName)) { column.setColumnName(String.valueOf(map.get(columnName))); } diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/ExampleDataHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/ExampleDataHelper.java new file mode 100644 index 00000000..c3eb9e52 --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/sqlparser/ExampleDataHelper.java @@ -0,0 +1,412 @@ +package com.mesalab.common.utils.sqlparser; + +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.CharsetUtil; +import com.alibaba.fastjson2.JSONPath; +import com.google.common.collect.Maps; +import com.mesalab.common.entity.DataTypeMapping; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.common.utils.RandomNumberGenerator; +import com.mesalab.qgw.constant.DataTypeConst; +import com.mesalab.qgw.constant.ExampleDataModeConst; +import com.mesalab.qgw.constant.MetaConst; +import com.mesalab.qgw.exception.QGWBusinessException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; + +import javax.validation.constraints.NotNull; +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +/** + * TODO + * + * @Classname ExampleDataHelper + * @Date 2024/3/4 18:49 + * @Author wWei + */ +@Slf4j +public class ExampleDataHelper { + + public static final String QUERY_TYPE_TIMESERIES = "Timeseries"; + public static final String QUERY_TYPE_GROUP_BY = "GroupBy"; + public static final String QUERY_TYPE_AGG_STATISTICS = "AggStatistics"; + public static final String QUERY_TYPE_OTHER = "Other"; + private static final String[] JSONPATH_SPECIAL_CHARS = {"/", ".", "[", "]", "'", "\""}; + + private ExampleDataHelper() { + } + + private static String exampleJsonData; + + static { + initJsonData(); + } + + + public static void initJsonData() { + String filePath; + try { + filePath = new File("").getCanonicalPath() + File.separator + "dat" + File.separator + "example_data.json"; + exampleJsonData = FileUtil.readString(filePath, CharsetUtil.UTF_8); + } catch (IOException e) { + log.error("Init Example Data Error: Read File Error, Error is:{}", e.getMessage()); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), e.getMessage())); + } + } + + /** + * Desc: TODO + * + * @param queryType Timeseries, TopN, GroupBy, AggStatistics, Other: Scan + * @param meta Column Meta + * @param sqlRowCount SQL结果行数 + * @param schemaDataDict Json Schema + * @return {@link List<Object>} + * @created by wWei + * @date 2024/3/6 16:59 + */ + public static List<Object> buildExampleData(@NotNull String queryType, List<Map<String, String>> meta, Long sqlRowCount, Map<String, List<Object>> schemaDataDict, Object queryTypeParameters) { + int rowCount = (sqlRowCount == null) ? RandomNumberGenerator.generateRangeRandomInt(10, 50) : Math.min(sqlRowCount.intValue(), 50); + List<LinkedHashMap<String, Object>> result; + switch (queryType) { + case QUERY_TYPE_TIMESERIES: + result = buildExampleDataOfTimeseries(queryTypeParameters.toString(), meta, sqlRowCount, schemaDataDict); + break; + case QUERY_TYPE_GROUP_BY: + result = buildExampleDataOfGroupBy(meta, rowCount, Boolean.parseBoolean(queryTypeParameters.toString()), schemaDataDict); + break; + case QUERY_TYPE_AGG_STATISTICS: + result = buildExampleDataOfMergeAggregation(meta, schemaDataDict); + break; + default: + result = buildExampleDataOfScan(meta, rowCount, schemaDataDict); + break; + } + return result.stream() + .map(map -> (Object) map) + .collect(Collectors.toList()); + } + + private static List<LinkedHashMap<String, Object>> buildExampleDataOfTimeseries(String timeGranLabel, List<Map<String, String>> meta, Long sqlRowCount, Map<String, List<Object>> schemaDataDict) { + Map<String, String> timeGranMeta = meta.stream() + .filter(item -> timeGranLabel.equals(item.get(MetaConst.META_NAME))) + .findFirst() + .orElse(Maps.newHashMap()); + List<LinkedHashMap<String, Object>> timeseriesList = new ArrayList<>(); + if (!timeGranMeta.isEmpty()) { + String timeGranDataType = timeGranMeta.get(MetaConst.META_DATA_TYPE); + timeGranDataType = timeGranDataType == null ? timeGranMeta.get(MetaConst.META_TYPE) : timeGranDataType; + Date currentDate = DateUtil.beginOfHour(new Date()); + int iteratorRowCount = (sqlRowCount == null) ? RandomNumberGenerator.generateRangeRandomInt(10, 50) : Math.min(sqlRowCount.intValue(), 50); + while (iteratorRowCount-- > 0) { + Object value = getDateTimeValue(timeGranDataType, iteratorRowCount, currentDate); + LinkedHashMap<String, Object> timeseries = Maps.newLinkedHashMap(); + timeseries.put(timeGranLabel, value); + timeseriesList.add(timeseries); + } + } + + int counter = 0; + List<LinkedHashMap<String, Object>> result = new ArrayList<>(); + for (int i = 0; i < meta.size(); i++) { + Map<String, String> item = meta.get(i); + if (MetaConst.META_CATEGORY_DIMENSION.equals(item.get(MetaConst.META_CATEGORY)) && !timeGranLabel.equals(item.get(MetaConst.META_NAME))) { + if (counter >= 1) { + timeseriesList = new ArrayList<>(result); + result = new ArrayList<>(); + } + counter++; + List exampleValue = getExampleValue(item, null, null, false, RandomNumberGenerator.generateRangeRandomInt(1, 5), schemaDataDict); + for (int i1 = 0; i1 < exampleValue.size(); i1++) { + List<LinkedHashMap<String, Object>> maps = new ArrayList<>(); + for (int j = 0; j < timeseriesList.size(); j++) { + LinkedHashMap<String, Object> copiedMap = new LinkedHashMap<>(timeseriesList.get(j)); + copiedMap.put(item.get(MetaConst.META_NAME), exampleValue.get(i1)); + maps.add(copiedMap); + } + result.addAll(maps); + } + } + } + + List<LinkedHashMap<String, Object>> finalResult = result.isEmpty() ? timeseriesList : result; + meta.stream().filter(item -> MetaConst.META_CATEGORY_METRIC.equals(item.get(MetaConst.META_CATEGORY))) + .forEach(item -> { + List exampleValue = getExampleValue(item, null, null, true, finalResult.size(), schemaDataDict); + for (int i = 0; i < finalResult.size(); i++) { + finalResult.get(i).put(item.get(MetaConst.META_NAME), exampleValue.get(i)); + } + }); + return (sqlRowCount == null) ? finalResult : finalResult.subList(0, Math.min(sqlRowCount.intValue(), finalResult.size())); + } + + private static List<LinkedHashMap<String, Object>> buildExampleDataOfGroupBy(List<Map<String, String>> meta, Integer rowCount, boolean isDesc, Map<String, List<Object>> schemaDataDict) { + for (Map<String, String> item : meta) { + if (inExampleData(item, schemaDataDict)) { + Integer exampleDataSize = getExampleDataSize(item, schemaDataDict); + rowCount = exampleDataSize == null ? rowCount : Math.min(rowCount, exampleDataSize); + } + } + List<LinkedHashMap<String, Object>> result = new ArrayList<>(); + for (int i = 0; i < rowCount; i++) { + result.add(Maps.newLinkedHashMap()); + } + Integer finalRowCount = rowCount; + meta.stream().filter(item -> MetaConst.META_CATEGORY_DIMENSION.equals(item.get(MetaConst.META_CATEGORY))) + .forEach(item -> { + List exampleValue = getExampleValue(item, null, null, false, finalRowCount, schemaDataDict); + for (int i = 0; i < result.size(); i++) { + result.get(i).put(item.get(MetaConst.META_NAME), exampleValue.get(i)); + } + }); + Integer finalRowCount1 = rowCount; + meta.stream().filter(item -> MetaConst.META_CATEGORY_METRIC.equals(item.get(MetaConst.META_CATEGORY))) + .forEach(item -> { + List exampleValue = getExampleValue(item, null, isDesc, true, finalRowCount1, schemaDataDict); + for (int i = 0; i < result.size(); i++) { + result.get(i).put(item.get(MetaConst.META_NAME), exampleValue.get(i)); + } + }); + return result; + } + + private static List<LinkedHashMap<String, Object>> buildExampleDataOfMergeAggregation(List<Map<String, String>> meta, Map<String, List<Object>> schemaDataDict) { + List<LinkedHashMap<String, Object>> result = new ArrayList<>(); + result.add(Maps.newLinkedHashMap()); + meta.stream().filter(item -> MetaConst.META_CATEGORY_METRIC.equals(item.get(MetaConst.META_CATEGORY))) + .forEach(item -> { + List exampleValue = getExampleValue(item, null, null, true, 1, schemaDataDict); + for (int i = 0; i < result.size(); i++) { + result.get(i).put(item.get(MetaConst.META_NAME), exampleValue.get(0)); + } + }); + return result; + } + + private static List<LinkedHashMap<String, Object>> buildExampleDataOfScan(List<Map<String, String>> meta, int rowCount, Map<String, List<Object>> schemaDataDict) { + for (Map<String, String> item : meta) { + if (inExampleData(item, schemaDataDict)) { + Integer exampleDataSize = getExampleDataSize(item, schemaDataDict); + rowCount = exampleDataSize == null ? rowCount : Math.min(rowCount, exampleDataSize); + } + } + List<LinkedHashMap<String, Object>> result = new ArrayList<>(); + for (int i = 0; i < rowCount; i++) { + result.add(Maps.newLinkedHashMap()); + } + int finalRowCount = rowCount; + meta.forEach(item -> { + List exampleValue = getExampleValue(item, null, null, true, finalRowCount, schemaDataDict); + for (int i = 0; i < result.size(); i++) { + result.get(i).put(item.get(MetaConst.META_NAME), exampleValue.get(i)); + } + }); + return result; + } + + private static List getExampleValue(Map<String, String> meta, Object initValue, Boolean isDesc, boolean canRepeat, int size, Map<String, List<Object>> schemaDataDict) { + + if (inExampleData(meta, schemaDataDict)) { + return getExampleDataValues(meta, canRepeat, size, schemaDataDict); + } + String dataType = meta.get(MetaConst.META_DATA_TYPE) == null ? meta.get(MetaConst.META_TYPE) : meta.get(MetaConst.META_DATA_TYPE); + + if (DataTypeMapping.LONG.equals(dataType) || DataTypeMapping.INT.equals(dataType)) { + return getExampleIntegerValues(null, isDesc, size); + } + if (DataTypeMapping.DOUBLE.equals(dataType) || DataTypeMapping.FLOAT.equals(dataType)) { + return getExampleDoubleValues(null, isDesc, size); + } + if (DataTypeMapping.BOOLEAN.equals(dataType)) { + return getExampleBooleanValues(size); + } + if (DataTypeMapping.DATE.equals(dataType) || DataTypeMapping.TIMESTAMP.equals(dataType) + || DataTypeConst.TIMESTAMP_FORMAT.equals(dataType) + || DataTypeConst.TIMESTAMP_MS_FORMAT.equals(dataType) || DataTypeConst.UNIX_TIMESTAMP.equals(dataType) + || DataTypeConst.UNIX_TIMESTAMP_MS.equals(dataType) + || DataTypeConst.DATE_TIME_64.equals(dataType) + ) { + return getExampleDateTimeValues(dataType, size); + } + return getExampleStrValues(size); + } + + private static List<Object> getExampleDateTimeValues(String dataType, int rowCount) { + List<Object> result = new ArrayList<>(); + Date currentDate = DateUtil.beginOfHour(new Date()); + while (rowCount-- > 0) { + result.add(getDateTimeValue(dataType, rowCount, currentDate)); + } + return result; + } + + private static Object getDateTimeValue(String dataType, int rowCount, Date currentDate) { + Object value; + switch (dataType) { + case DataTypeConst.TIMESTAMP: + value = DateUtil.format(DateUtil.offsetHour(DateUtil.beginOfHour(currentDate), -rowCount), DatePattern.UTC_WITH_ZONE_OFFSET_PATTERN); + break; + case DataTypeConst.TIMESTAMP_MS_FORMAT: + value = DateUtil.format(DateUtil.offsetHour(currentDate, -rowCount), DatePattern.NORM_DATETIME_MS_FORMAT); + break; + case DataTypeConst.UNIX_TIMESTAMP: + value = DateUtil.offsetHour(currentDate, -rowCount).toInstant().getEpochSecond(); + break; + case DataTypeConst.UNIX_TIMESTAMP_MS: + value = DateUtil.offsetHour(currentDate, -rowCount).toInstant().toEpochMilli(); + break; + case DataTypeConst.DATE: + value = DateUtil.format(DateUtil.offsetHour(currentDate, -rowCount), DatePattern.NORM_DATE_PATTERN); + break; + case DataTypeConst.DATE_TIME_64: + value = DateUtil.format(DateUtil.offsetHour(currentDate, -rowCount), DatePattern.NORM_DATETIME_MS_FORMAT); + break; + default: + value = DateUtil.format(DateUtil.offsetHour(currentDate, -rowCount), DatePattern.NORM_DATETIME_FORMAT); + break; + } + return value; + } + + private static List<Boolean> getExampleBooleanValues(int size) { + List<Boolean> result = new ArrayList<>(); + for (int i = 0; i < size; i++) { + result.add(new Random().nextBoolean()); + } + return result; + } + + private static List<Double> getExampleDoubleValues(Double initValue, Boolean increase, int size) { + List<Double> result = new ArrayList<>(); + Double relativeV = initValue; + for (int i = 0; i < size; i++) { + relativeV = RandomNumberGenerator.generateRelativeRandomDouble(relativeV, increase); + result.add(relativeV); + } + return result; + } + + private static List<Integer> getExampleIntegerValues(Integer initValue, Boolean idDesc, int size) { + List<Integer> result = new ArrayList<>(); + Integer relativeV = initValue; + for (int i = 0; i < size; i++) { + relativeV = RandomNumberGenerator.generateRelativeRandomInt(relativeV, idDesc); + result.add(relativeV); + } + return result; + } + + private static List<Object> getExampleDataValues(Map<String, String> meta, boolean canRepeat, int size, Map<String, List<Object>> schemaDataDict) { + List<Object> result = new ArrayList<>(); + + if (schemaDataDict.containsKey(meta.get(MetaConst.META_FIELD_NAME))) { + List<Object> list = schemaDataDict.get(meta.get(MetaConst.META_FIELD_NAME)); + buildEnumDataList(canRepeat, size, result, list); + return result; + } + + + String dataType = meta.get(MetaConst.META_DATA_TYPE) == null ? meta.get(MetaConst.META_TYPE) : meta.get(MetaConst.META_DATA_TYPE); + String path = "$." + dataType; + Object mode = JSONPath.extract(exampleJsonData, path + ".mode"); + if (mode == null) { + for (int i = 0; i < size; i++) { + result.add(null); + } + } else if (ExampleDataModeConst.RANGE.equalsIgnoreCase(mode.toString())) { + Integer start = (Integer) JSONPath.extract(exampleJsonData, path + ".start"); + Integer end = (Integer) JSONPath.extract(exampleJsonData, path + ".end"); + for (int i = 0; i < size; i++) { + int v = RandomNumberGenerator.generateRangeRandomInt(start, end); + result.add(v); + } + } else if (ExampleDataModeConst.ENUM.equalsIgnoreCase(mode.toString())) { + List list = (List) JSONPath.extract(exampleJsonData, path + ".values"); + buildEnumDataList(canRepeat, size, result, list); + } else if (ExampleDataModeConst.SEQUENCE.equalsIgnoreCase(mode.toString())) { + List list = (List) JSONPath.extract(exampleJsonData, path + ".values"); + result.addAll(list.subList(0, size)); + } + return result; + } + + private static void buildEnumDataList(boolean canRepeat, int size, List<Object> result, List<Object> list) { + if (canRepeat) { + for (int i = 0; i < size; i++) { + int index = RandomNumberGenerator.generateRangeRandomInt(0, list.size() - 1); + result.add(list.get(index)); + } + } else { + Collections.shuffle(list); + result.addAll(list.subList(0, Math.min(size, list.size() ))); + } + } + + private static Integer getExampleDataSize(Map<String, String> meta, Map<String, List<Object>> schemaDataDict) { + if (schemaDataDict.containsKey(meta.get(MetaConst.META_FIELD_NAME))) { + return schemaDataDict.get(meta.get(MetaConst.META_FIELD_NAME)).size(); + } + + String dataType = meta.get(MetaConst.META_DATA_TYPE) == null ? meta.get(MetaConst.META_TYPE) : meta.get(MetaConst.META_DATA_TYPE); + String path = "$." + dataType; + Object mode = JSONPath.extract(exampleJsonData, path + ".mode"); + if (ExampleDataModeConst.ENUM.equalsIgnoreCase(mode.toString())) { + List list = (List) JSONPath.extract(exampleJsonData, path + ".values"); + return list.size(); + } else if (ExampleDataModeConst.SEQUENCE.equalsIgnoreCase(mode.toString())) { + List list = (List) JSONPath.extract(exampleJsonData, path + ".values"); + return list.size(); + } + return null; + } + + + /** + * Desc: Generate random data of type List<Object> + * + * @param size result size + * @return {@link List<Object>} + * @created by wWei + * @date 2024/3/9 15:00 + */ + private static List<Object> getExampleStrValues(int size) { + List<Object> result = new ArrayList<>(); + for (int i = 0; i < size; i++) { + result.add(RandomStringUtils.randomAlphanumeric(3, 10)); + } + return result; + } + + /** + * Desc: Determine if the current result column is in the example data file or schema dictionary data + * + * @param meta meta information + * @param schemaDataDict schema data dictionary + * @return {@link boolean} + * @created by wWei + * @date 2024/3/9 15:02 + */ + private static boolean inExampleData(Map<String, String> meta, Map<String, List<Object>> schemaDataDict) { + if (schemaDataDict == null) { + return false; + } + if (schemaDataDict.containsKey(meta.get(MetaConst.META_FIELD_NAME))) { + return true; + } + + String dataType = meta.get(MetaConst.META_DATA_TYPE) == null ? meta.get(MetaConst.META_TYPE) : meta.get(MetaConst.META_DATA_TYPE); + // 对特殊字符进行转义处理 + for (String specialChar : JSONPATH_SPECIAL_CHARS) { + dataType = dataType.replace(specialChar, "\\" + specialChar); + } + String path = "$." + dataType; + return JSONPath.extract(exampleJsonData, path) != null; + } +} diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/ExpressionColumnCollectAdapter.java b/src/main/java/com/mesalab/common/utils/sqlparser/ExpressionColumnCollectAdapter.java new file mode 100644 index 00000000..acea2d6f --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/sqlparser/ExpressionColumnCollectAdapter.java @@ -0,0 +1,29 @@ +package com.mesalab.common.utils.sqlparser; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import lombok.Data; +import net.sf.jsqlparser.expression.*; +import net.sf.jsqlparser.schema.Column; + +import java.util.List; +import java.util.Set; + + +/** + * @Classname ExpressionColumnCollectAdapter + * @Date 2021/7/20 3:39 下午 + * @Author wWei + */ +@Data +public class ExpressionColumnCollectAdapter extends ExpressionVisitorAdapter { + + public Set<String> columns = Sets.newHashSet(); + + @Override + public void visit(Column column) { + if (column != null) { + this.columns.add(column.getColumnName()); + } + } +} diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/FunctionsMergeHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/FunctionsMergeHelper.java new file mode 100644 index 00000000..adc16c42 --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/sqlparser/FunctionsMergeHelper.java @@ -0,0 +1,382 @@ +package com.mesalab.common.utils.sqlparser; + +import lombok.SneakyThrows; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.operators.arithmetic.Division; +import net.sf.jsqlparser.expression.operators.arithmetic.Multiplication; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.*; + +import java.util.*; + +/** + * TODO + * + * @Classname FunctionsMergeHelper + * @Date 2024/2/20 10:42 + * @Author wWei + */ +public class FunctionsMergeHelper { + + private static final String FROM_UNIXTIME = "FROM_UNIXTIME"; + private static final String TIME_FLOOR_WITH_FILL = "TIME_FLOOR_WITH_FILL"; + private static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP"; + private static final String TIME_FORMAT = "TIME_FORMAT"; + private static final String DATE_TIME_FORMAT_PATTERN = "'yyyy-MM-dd HH:mm:ss'"; + private static final String TIME_FLOOR = "TIME_FLOOR"; + private static final String MILLIS_TO_TIMESTAMP = "MILLIS_TO_TIMESTAMP"; + private static final String TIMESTAMP_TO_MILLIS = "TIMESTAMP_TO_MILLIS"; + + private FunctionsMergeHelper() { + } + + /** + * Desc: Merge the TIME_FLOOR_WITH_FILL functions in the SQL statement + * 1. FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), *[, *])) ==> + * 1.1 Select Item: TIME_FORMAT(TIME_FLOOR(column, *), 'yyyy-MM-dd HH:mm:ss') + * 1.2 GroupBy or Order By Expr: TIME_FLOOR(column, *) + * + * 2. FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(column, *[, *])) ==> + * 2.1 Select Item: TIME_FORMAT(TIME_FLOOR(MILLIS_TO_TIMESTAMP(column * 1000), *), 'yyyy-MM-dd HH:mm:ss') + * 2.2 GroupBy or Order By Expr: TIME_FLOOR(MILLIS_TO_TIMESTAMP(column * 1000), *)] + * + * 3. TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), *[, *]) ==> + * 3.1 Select Item: TIMESTAMP_TO_MILLIS(TIME_FLOOR(column, *)) / 1000 + * 3.2 GroupBy or Order By Expr: TIME_FLOOR(column, *) + * + * 4. TIME_FLOOR_WITH_FILL(column, *[, *]) ==> + * 4.1 Select Item: TIMESTAMP_TO_MILLIS(TIME_FLOOR(MILLIS_TO_TIMESTAMP(column * 1000), *)) / 1000 + * 4.2 GroupBy or Order By Expr: TIME_FLOOR(MILLIS_TO_TIMESTAMP(column * 1000), *) + * + * @param sql + * @return {@link String} + * @created by wWei + * @date 2024/2/24 14:49 + */ + public static String build(String sql) throws JSQLParserException { + Select select = (Select) CCJSqlParserUtil.parse(sql); + select.getSelectBody().accept(new SelectAdapter()); + return select.toString(); + } + + private static class SelectAdapter extends SelectVisitorAdapter { + + @SneakyThrows + @Override + public void visit(PlainSelect plainSelect) { + + Map<String, String> aliasExpr = new HashMap<>(); + List<SelectItem> selectItems = plainSelect.getSelectItems(); + if (selectItems != null) { + for (SelectItem selectItem : selectItems) { + SelectItemAdapter selectItemAdapter = new SelectItemAdapter(); + selectItem.accept(selectItemAdapter); + Map<String, String> itemAliasExpr = selectItemAdapter.getAliasExpr(); + if (!itemAliasExpr.isEmpty()) { + aliasExpr.putAll(itemAliasExpr); + } + } + } + + GroupByElement groupBy = plainSelect.getGroupBy(); + if (groupBy != null && groupBy.getGroupByExpressionList() != null && !groupBy.getGroupByExpressionList().getExpressions().isEmpty()) { + groupBy.accept(new GroupByAdapter()); + } + List<OrderByElement> orderByElements = plainSelect.getOrderByElements(); + if (orderByElements != null) { + for (OrderByElement orderByElement : orderByElements) { + if (aliasExpr.containsKey(orderByElement.getExpression().toString())) { + Expression expression = CCJSqlParserUtil.parseExpression(aliasExpr.get(orderByElement.getExpression().toString())); + orderByElement.setExpression(expression); + } + orderByElement.accept(new OrderByAdapter()); + } + } + + FromItem fromItem = plainSelect.getFromItem(); + if (fromItem != null) { + fromItem.accept(new FromItemAdapter()); + } + + } + + @Override + public void visit(SetOperationList setOpList) { + for (SelectBody select : setOpList.getSelects()) { + if (select instanceof PlainSelect) { + select.accept(this); + } + } + } + } + + private static Function createFunction(String funName, List<Expression> params) { + Function function = new Function(); + function.setName(funName); + function.setParameters(new ExpressionList(params)); + return function; + } + + private static boolean isUnixTimestampFunction(Expression expression) { + if (!(expression instanceof Function)) { + return false; + } + Function function = (Function) expression; + return function.getName().equalsIgnoreCase(UNIX_TIMESTAMP) + && function.getParameters().getExpressions().size() == 1; + } + + private static boolean isTimeFloorWithFillFunction(Expression expression) { + if (!(expression instanceof Function)) { + return false; + } + Function function = (Function) expression; + return TIME_FLOOR_WITH_FILL.equalsIgnoreCase(function.getName()) + && function.getParameters().getExpressions().size() >= 2 + && function.getParameters().getExpressions().size() <= 3; + } + + private static boolean isFromUnixtimeFunction(Expression expression) { + if (!(expression instanceof Function)) { + return false; + } + Function function = (Function) expression; + return function.getName().equalsIgnoreCase(FROM_UNIXTIME) + && function.getParameters().getExpressions().size() == 1; + } + + + private static class SelectItemAdapter extends SelectItemVisitorAdapter { + + private Map<String, String> aliasExpr = new HashMap<>(); + + public Map<String, String> getAliasExpr() { + return aliasExpr; + } + + @Override + public void visit(SelectExpressionItem selectExpressionItem) { + Expression expression = selectExpressionItem.getExpression(); + if (!(expression instanceof Function)) { + return; + } + Function mainFunction = (Function) expression; + if (isFromUnixtimeFunction(mainFunction)) { + Expression fromUnixtimeArg0 = mainFunction.getParameters().getExpressions().get(0); + if (isTimeFloorWithFillFunction(fromUnixtimeArg0)) { + Function timeFloorWithFillFun = (Function) fromUnixtimeArg0; + Expression timeFloorWithFillFunArg0 = timeFloorWithFillFun.getParameters().getExpressions().get(0); + Expression timeFloorWithFillFunArg1 = timeFloorWithFillFun.getParameters().getExpressions().get(1); + if (isUnixTimestampFunction(timeFloorWithFillFunArg0) && ((Function) timeFloorWithFillFunArg0).getParameters().getExpressions().get(0) instanceof Column) { + if (selectExpressionItem.getAlias() != null) { + aliasExpr.put(selectExpressionItem.getAlias().getName(), mainFunction.toString()); + } + + Function unixTimestampFun = (Function) timeFloorWithFillFunArg0; + List<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(unixTimestampFun.getParameters().getExpressions().get(0), timeFloorWithFillFunArg1)); + Function timeFloor = createFunction(TIME_FLOOR, timeFloorArgs); + mainFunction.setName(TIME_FORMAT); + List<Expression> timeFormatArgs = new ArrayList<>(Arrays.asList(timeFloor, new StringValue(DATE_TIME_FORMAT_PATTERN))); + mainFunction.setParameters(new ExpressionList(timeFormatArgs)); + } else if (timeFloorWithFillFunArg0 instanceof Column) { + if (selectExpressionItem.getAlias() != null) { + aliasExpr.put(selectExpressionItem.getAlias().getName(), mainFunction.toString()); + } + Multiplication multiplication = new Multiplication(); + multiplication.setLeftExpression(timeFloorWithFillFunArg0); + multiplication.setRightExpression(new LongValue(1000)); + + List<Expression> millisToTimestampArgs = new ArrayList<>(Collections.singletonList(multiplication)); + Function millsToTimestamp = createFunction(MILLIS_TO_TIMESTAMP, millisToTimestampArgs); + + ArrayList<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(millsToTimestamp, timeFloorWithFillFunArg1)); + Function timeFloorFun = createFunction(TIME_FLOOR, timeFloorArgs); + + mainFunction.setName(TIME_FORMAT); + List<Expression> timeFormatArgs = new ArrayList<>(Arrays.asList(timeFloorFun, new StringValue(DATE_TIME_FORMAT_PATTERN))); + mainFunction.setParameters(new ExpressionList(timeFormatArgs)); + } + } + } else if (isTimeFloorWithFillFunction(mainFunction)) { + Expression timeFloorWithFillFunArg0 = mainFunction.getParameters().getExpressions().get(0); + Expression timeFloorWithFillFunArg1 = mainFunction.getParameters().getExpressions().get(1); + if (isUnixTimestampFunction(timeFloorWithFillFunArg0) && ((Function) timeFloorWithFillFunArg0).getParameters().getExpressions().get(0) instanceof Column) { + + if (selectExpressionItem.getAlias() != null) { + aliasExpr.put(selectExpressionItem.getAlias().getName(), mainFunction.toString()); + } + Function unixTimestampFun = (Function) timeFloorWithFillFunArg0; + + List<Expression> expressions = new ArrayList<>(Arrays.asList(unixTimestampFun.getParameters().getExpressions().get(0), timeFloorWithFillFunArg1)); + Function timeFloor = createFunction(TIME_FLOOR, expressions); + + List<Expression> timestampToMillsArgs = new ArrayList<>(Collections.singletonList(timeFloor)); + Function timestampToMillis = createFunction(TIMESTAMP_TO_MILLIS, timestampToMillsArgs); + + Division division = new Division(); + division.setLeftExpression(timestampToMillis); + division.setRightExpression(new LongValue(1000)); + + selectExpressionItem.setExpression(division); + } else if (timeFloorWithFillFunArg0 instanceof Column) { + if (selectExpressionItem.getAlias() != null) { + aliasExpr.put(selectExpressionItem.getAlias().getName(), mainFunction.toString()); + } + + Multiplication multiplication = new Multiplication(); + multiplication.setLeftExpression(timeFloorWithFillFunArg0); + multiplication.setRightExpression(new LongValue(1000)); + + List<Expression> millisToTimestampArgs = new ArrayList<>(Collections.singletonList(multiplication)); + Function millsToTimestamp = createFunction(MILLIS_TO_TIMESTAMP, millisToTimestampArgs); + + List<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(millsToTimestamp, timeFloorWithFillFunArg1)); + Function timeFloor = createFunction(TIME_FLOOR, timeFloorArgs); + + List<Expression> timestampToMillsArgs = new ArrayList<>(Collections.singletonList(timeFloor)); + Function timestampToMills = createFunction(TIMESTAMP_TO_MILLIS, timestampToMillsArgs); + + Division division = new Division(); + division.setLeftExpression(timestampToMills); + division.setRightExpression(new LongValue(1000)); + + selectExpressionItem.setExpression(division); + } + } + } + + } + + private static class FromItemAdapter extends FromItemVisitorAdapter { + @Override + public void visit(SubSelect subSelect) { + SelectBody selectBody = subSelect.getSelectBody(); + selectBody.accept(new SelectAdapter()); + } + } + + private static class GroupByAdapter implements GroupByVisitor { + @Override + public void visit(GroupByElement groupByElement) { + if (groupByElement.getGroupByExpressionList() == null + || groupByElement.getGroupByExpressionList().getExpressions() == null) { + return; + } + for (Expression expression : groupByElement.getGroupByExpressionList().getExpressions()) { + if (!(expression instanceof Function)) { + continue; + } + Function mainFunction = (Function) expression; + if (isFromUnixtimeFunction(expression)) { + Expression fromUnixtimeArg0 = mainFunction.getParameters().getExpressions().get(0); + if (isTimeFloorWithFillFunction(fromUnixtimeArg0)) { + Function timeFloorWithFillFun = (Function) fromUnixtimeArg0; + Expression timeFloorWithFillFunArg0 = timeFloorWithFillFun.getParameters().getExpressions().get(0); + Expression timeFloorWithFillFunArg1 = timeFloorWithFillFun.getParameters().getExpressions().get(1); + if (isUnixTimestampFunction(timeFloorWithFillFunArg0) && ((Function) timeFloorWithFillFunArg0).getParameters().getExpressions().get(0) instanceof Column) { + Function unixTimestampFun = (Function) timeFloorWithFillFunArg0; + mainFunction.setName(TIME_FLOOR); + List<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(unixTimestampFun.getParameters().getExpressions().get(0), timeFloorWithFillFunArg1)); + mainFunction.setParameters(new ExpressionList(timeFloorArgs)); + } else if (timeFloorWithFillFunArg0 instanceof Column) { + Multiplication multiplication = new Multiplication(); + multiplication.setLeftExpression(timeFloorWithFillFunArg0); + multiplication.setRightExpression(new LongValue(1000)); + + List<Expression> millisToTimestampArgs = new ArrayList<>(Collections.singletonList(multiplication)); + Function millsToTimestamp = createFunction(MILLIS_TO_TIMESTAMP, millisToTimestampArgs); + + mainFunction.setName(TIME_FLOOR); + ArrayList<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(millsToTimestamp, timeFloorWithFillFunArg1)); + mainFunction.setParameters(new ExpressionList(timeFloorArgs)); + } + } + } else if (isTimeFloorWithFillFunction(mainFunction)) { + Expression timeFloorWithFillFunArg0 = mainFunction.getParameters().getExpressions().get(0); + Expression timeFloorWithFillFunArg1 = mainFunction.getParameters().getExpressions().get(1); + if (isUnixTimestampFunction(timeFloorWithFillFunArg0) && ((Function) timeFloorWithFillFunArg0).getParameters().getExpressions().get(0) instanceof Column) { + Function unixTimestampFun = (Function) timeFloorWithFillFunArg0; + mainFunction.setName(TIME_FLOOR); + List<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(unixTimestampFun.getParameters().getExpressions().get(0), timeFloorWithFillFunArg1)); + mainFunction.setParameters(new ExpressionList(timeFloorArgs)); + } else if (timeFloorWithFillFunArg0 instanceof Column) { + Multiplication multiplication = new Multiplication(); + multiplication.setLeftExpression(timeFloorWithFillFunArg0); + multiplication.setRightExpression(new LongValue(1000)); + + List<Expression> millisToTimestampArgs = new ArrayList<>(Collections.singletonList(multiplication)); + Function millsToTimestamp = createFunction(MILLIS_TO_TIMESTAMP, millisToTimestampArgs); + + mainFunction.setName(TIME_FLOOR); + List<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(millsToTimestamp, timeFloorWithFillFunArg1)); + mainFunction.setParameters(new ExpressionList(timeFloorArgs)); + } + } + } + } + } + + private static class OrderByAdapter implements OrderByVisitor { + @Override + public void visit(OrderByElement orderBy) { + if (orderBy.getExpression() == null) { + return; + } + if (!(orderBy.getExpression() instanceof Function)) { + return; + } + Function mainFunction = (Function) orderBy.getExpression(); + if (isFromUnixtimeFunction(mainFunction)) { + Expression fromUnixtimeArg0 = mainFunction.getParameters().getExpressions().get(0); + if (isTimeFloorWithFillFunction(fromUnixtimeArg0)) { + Function timeFloorWithFillFun = (Function) fromUnixtimeArg0; + Expression timeFloorWithFillFunArg0 = timeFloorWithFillFun.getParameters().getExpressions().get(0); + Expression timeFloorWithFillFunArg1 = timeFloorWithFillFun.getParameters().getExpressions().get(1); + if (isUnixTimestampFunction(timeFloorWithFillFunArg0) && ((Function) timeFloorWithFillFunArg0).getParameters().getExpressions().get(0) instanceof Column) { + Function unixTimestampFun = (Function) timeFloorWithFillFunArg0; + mainFunction.setName(TIME_FLOOR); + List<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(unixTimestampFun.getParameters().getExpressions().get(0), timeFloorWithFillFunArg1)); + mainFunction.setParameters(new ExpressionList(timeFloorArgs)); + } else if (timeFloorWithFillFunArg0 instanceof Column) { + Multiplication multiplication = new Multiplication(); + multiplication.setLeftExpression(timeFloorWithFillFunArg0); + multiplication.setRightExpression(new LongValue(1000)); + + List<Expression> millisToTimestampArgs = new ArrayList<>(Collections.singletonList(multiplication)); + Function millsToTimestamp = createFunction(MILLIS_TO_TIMESTAMP, millisToTimestampArgs); + + mainFunction.setName(TIME_FLOOR); + ArrayList<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(millsToTimestamp, timeFloorWithFillFunArg1)); + mainFunction.setParameters(new ExpressionList(timeFloorArgs)); + } + } + } else if (isTimeFloorWithFillFunction(mainFunction)) { + Expression timeFloorWithFillFunArg0 = mainFunction.getParameters().getExpressions().get(0); + Expression timeFloorWithFillFunArg1 = mainFunction.getParameters().getExpressions().get(1); + if (isUnixTimestampFunction(timeFloorWithFillFunArg0) && ((Function) timeFloorWithFillFunArg0).getParameters().getExpressions().get(0) instanceof Column) { + Function unixTimestampFun = (Function) timeFloorWithFillFunArg0; + mainFunction.setName(TIME_FLOOR); + List<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(unixTimestampFun.getParameters().getExpressions().get(0), timeFloorWithFillFunArg1)); + mainFunction.setParameters(new ExpressionList(timeFloorArgs)); + } else if (timeFloorWithFillFunArg0 instanceof Column) { + Multiplication multiplication = new Multiplication(); + multiplication.setLeftExpression(timeFloorWithFillFunArg0); + multiplication.setRightExpression(new LongValue(1000)); + + List<Expression> millisToTimestampArgs = new ArrayList<>(Collections.singletonList(multiplication)); + Function millsToTimestamp = createFunction(MILLIS_TO_TIMESTAMP, millisToTimestampArgs); + + mainFunction.setName(TIME_FLOOR); + List<Expression> timeFloorArgs = new ArrayList<>(Arrays.asList(millsToTimestamp, timeFloorWithFillFunArg1)); + mainFunction.setParameters(new ExpressionList(timeFloorArgs)); + } + } + } + } +} diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/QueryTypeHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/QueryTypeHelper.java new file mode 100644 index 00000000..ecb0bfad --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/sqlparser/QueryTypeHelper.java @@ -0,0 +1,92 @@ +package com.mesalab.common.utils.sqlparser; + +import com.mesalab.qgw.dialect.ClickHouseDialect; +import com.mesalab.qgw.dialect.DruidDialect; +import com.mesalab.qgw.model.basic.SelectStatement; +import com.mesalab.qgw.model.basic.udf.TIME_FLOOR_WITH_FILL; +import com.mesalab.qgw.model.basic.udf.UDF; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.statement.select.OrderByElement; +import net.sf.jsqlparser.statement.select.SelectExpressionItem; +import net.sf.jsqlparser.statement.select.SelectItem; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * TODO + * + * @Classname QueryTypeHelper + * @Date 2024/3/11 14:43 + * @Author wWei + */ +public class QueryTypeHelper { + + private QueryTypeHelper() { + } + + public static String determineQueryType(SelectStatement dbSelectStatement) { + Set<UDF> udfSet = dbSelectStatement.getUdfSet(); + if (udfSet != null && !udfSet.isEmpty()) { + for (UDF udf : udfSet) { + if (udf instanceof TIME_FLOOR_WITH_FILL) { + return ExampleDataHelper.QUERY_TYPE_TIMESERIES; + } + } + } + + Map<String, String> groupDimension = dbSelectStatement.getGroupDimension(); + if (groupDimension != null && !groupDimension.isEmpty()) { + return ExampleDataHelper.QUERY_TYPE_GROUP_BY; + } + + List<SelectItem> selectItems = dbSelectStatement.getSelectItems(); + if (selectItems != null && !selectItems.isEmpty()) { + for (SelectItem selectItem : selectItems) { + if (selectItem instanceof SelectExpressionItem) { + SelectExpressionItem selectExpressionItem = ((SelectExpressionItem) selectItem); + if (selectExpressionItem.getExpression() instanceof Function) { + Function function = (Function) selectExpressionItem.getExpression(); + if (SQLFunctionUtil.pAggregateFunStandard.matcher(function.toString()).find() + || ClickHouseDialect.pAggregateFunSpecificOfCK.matcher(function.toString()).find() + || DruidDialect.pAggregateFunSpecificOfDruid.matcher(function.toString()).find()) { + return ExampleDataHelper.QUERY_TYPE_AGG_STATISTICS; + } + } + } + } + } + return ExampleDataHelper.QUERY_TYPE_OTHER; + } + + public static Object determineParam(SelectStatement dbSelectStatement, String queryType) { + if (ExampleDataHelper.QUERY_TYPE_TIMESERIES.equals(queryType)) { + TIME_FLOOR_WITH_FILL udfElements = dbSelectStatement.getUdfSet().stream() + .filter(udf -> udf instanceof TIME_FLOOR_WITH_FILL) + .map(udf -> (TIME_FLOOR_WITH_FILL) udf) + .findFirst() + .orElse(null); + + if (udfElements != null) { + String name = udfElements.getName(); + String timestamp = udfElements.getTimestamp(); + Map<String, String> groupDimension = dbSelectStatement.getGroupDimension(); + for (String groupKey : groupDimension.keySet()) { + if (groupDimension.get(groupKey).contains(name) && groupDimension.get(groupKey).contains(timestamp)) { + return groupKey; + } + } + } + } else if (ExampleDataHelper.QUERY_TYPE_GROUP_BY.equals(queryType)) { + List<OrderByElement> orderByElements = dbSelectStatement.getOrderByElements(); + if (orderByElements != null && !orderByElements.isEmpty()) { + return !orderByElements.get(0).isAsc(); + } else { + return false; + } + } + + return null; + } +} diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/SQLFunctionUtil.java b/src/main/java/com/mesalab/common/utils/sqlparser/SQLFunctionUtil.java index 8c0d7351..c65cc6f8 100644 --- a/src/main/java/com/mesalab/common/utils/sqlparser/SQLFunctionUtil.java +++ b/src/main/java/com/mesalab/common/utils/sqlparser/SQLFunctionUtil.java @@ -1,17 +1,19 @@ package com.mesalab.common.utils.sqlparser; +import cn.hutool.core.util.NumberUtil; +import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.URLUtil; import com.alibaba.fastjson2.JSON; import com.geedgenetworks.utils.Encodes; import com.google.common.base.Splitter; import com.google.common.collect.Maps; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.DBEngineType; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.utils.SpringContextUtil; import com.mesalab.common.utils.StringUtil; import com.mesalab.common.utils.TimeZoneUtil; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; +import com.mesalab.common.exception.CommonErrorCode; import com.mesalab.qgw.model.basic.HttpConfig; import com.mesalab.qgw.model.basic.ClickHouseHttpSource; import com.mesalab.qgw.model.basic.DruidIoHttpSource; @@ -24,7 +26,9 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.apache.commons.lang3.StringUtils; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; +import org.joda.time.Period; +import javax.validation.constraints.NotNull; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; @@ -32,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * @author wangwei @@ -39,6 +44,7 @@ import java.util.regex.Pattern; */ public class SQLFunctionUtil { + private static final boolean CONVERT_DEFAULT_TIMEZONE_INTO_UTC; public static final String TIME_FLOOR_WITH_FILL = "TIME_FLOOR_WITH_FILL"; public static final String IP_TO_GEO = "IP_TO_GEO"; public static final String IP_TO_CITY = "IP_TO_CITY"; @@ -47,6 +53,8 @@ public class SQLFunctionUtil { public static final String IP_TO_ASN = "IP_TO_ASN"; public static final String IP_TO_ASN_DETAIL = "IP_TO_ASN_DETAIL"; public static final String IP_TO_ASN_ORG = "IP_TO_ASN_ORG"; + public static final String ROLLUP = "ROLLUP"; + public static final String RATE = "RATE"; public static final String MAX_DURATION = "MAX_DURATION"; public static final String MEDIAN_HDR = "MEDIAN_HDR"; public static final String QUANTILE_HDR = "QUANTILE_HDR"; @@ -68,6 +76,7 @@ public class SQLFunctionUtil { */ public static final Pattern pTimeFloorWithFill = Pattern.compile("\\b" + TIME_FLOOR_WITH_FILL + "\\(", Pattern.CASE_INSENSITIVE); public static final Pattern pMaxDuration = Pattern.compile("\\b" + MAX_DURATION + "\\(", Pattern.CASE_INSENSITIVE); + public static final Pattern pRate = Pattern.compile("\\b" + RATE + "\\(", Pattern.CASE_INSENSITIVE); public static final Pattern pIpToGeo = Pattern.compile("\\b" + IP_TO_GEO + "\\(", Pattern.CASE_INSENSITIVE); public static final Pattern pIpToCity = Pattern.compile("\\b" + IP_TO_CITY + "\\(", Pattern.CASE_INSENSITIVE); public static final Pattern pIpToCountry = Pattern.compile("\\b" + IP_TO_COUNTRY + "\\(", Pattern.CASE_INSENSITIVE); @@ -88,38 +97,40 @@ public class SQLFunctionUtil { public static final Pattern pApproxCountDistinctHlld = Pattern.compile("\\b" + APPROX_COUNT_DISTINCT_HLLD + "\\(", Pattern.CASE_INSENSITIVE); - public static final HashMap<String, String> functions = Maps.newHashMap(); + public static final HashMap<String, String> federationUDFFunctions = Maps.newHashMap(); private static final Map<String, String> distributedGroupByFunction = Maps.newHashMap(); public static final Map<String, String> timeSlicingFunction = Maps.newHashMap(); static { - functions.put(TIME_FLOOR_WITH_FILL, null); - functions.put(IP_TO_GEO, null); - functions.put(IP_TO_CITY, null); - functions.put(IP_TO_COUNTRY, null); - functions.put(IP_TO_ISP, null); - functions.put(IP_TO_ASN, null); - functions.put(IP_TO_ASN_DETAIL, null); - functions.put(IP_TO_ASN_ORG, null); - distributedGroupByFunction.put("count","sum"); - distributedGroupByFunction.put("sum","sum"); - distributedGroupByFunction.put("avg","avg"); - distributedGroupByFunction.put("min","min"); - distributedGroupByFunction.put("max","max"); - distributedGroupByFunction.put("median","median"); - timeSlicingFunction.put("count","sum"); - timeSlicingFunction.put("sum","sum"); - timeSlicingFunction.put("avg","avg"); - timeSlicingFunction.put("min","min"); - timeSlicingFunction.put("max","max"); + federationUDFFunctions.put(TIME_FLOOR_WITH_FILL, null); + federationUDFFunctions.put(IP_TO_GEO, null); + federationUDFFunctions.put(IP_TO_CITY, null); + federationUDFFunctions.put(IP_TO_COUNTRY, null); + federationUDFFunctions.put(IP_TO_ISP, null); + federationUDFFunctions.put(IP_TO_ASN, null); + federationUDFFunctions.put(IP_TO_ASN_DETAIL, null); + federationUDFFunctions.put(IP_TO_ASN_ORG, null); + federationUDFFunctions.put(ROLLUP, null); + distributedGroupByFunction.put("count", "sum"); + distributedGroupByFunction.put("sum", "sum"); + distributedGroupByFunction.put("avg", "avg"); + distributedGroupByFunction.put("min", "min"); + distributedGroupByFunction.put("max", "max"); + distributedGroupByFunction.put("median", "median"); + timeSlicingFunction.put("count", "sum"); + timeSlicingFunction.put("sum", "sum"); + timeSlicingFunction.put("avg", "avg"); + timeSlicingFunction.put("min", "min"); + timeSlicingFunction.put("max", "max"); + CONVERT_DEFAULT_TIMEZONE_INTO_UTC = clickHouseHttpSource.isConvertDefaultTimezoneIntoUTC(); } /** * 标准函数 */ //time functions - public static final Pattern pAggregateFunStandard = Pattern.compile("\\b(count|min|max|sum|avg|any|stddevPop|stddevSamp|varPop|varSamp|covarPop|covarSamp)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static final Pattern pAggregateFunStandard = Pattern.compile("\\b(count|min|max|sum|avg|any|rate|stddevPop|stddevSamp|varPop|varSamp|covarPop|covarSamp|MEDIAN|QUANTILE|MAX_DURATION|MEDIAN_HDR|QUANTILE_HDR|PERCENTILES_HDR|COUNT_DISTINCT|APPROX_COUNT_DISTINCT_HLLD)\\s*\\(", Pattern.CASE_INSENSITIVE); public static final Pattern pDateFormat = Pattern.compile("\\W(FROM_UNIXTIME|DATE_FORMAT|STR_TO_DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); public static final Pattern pGroupByFormat = Pattern.compile("(SELECT\\s+|GROUP\\s*BY\\s+|,\\s*)(FROM_UNIXTIME|DATE_FORMAT|STR_TO_DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); public static final Pattern pFromUnixTimeMillis = Pattern.compile("\\WFROM_UNIXTIME_MILLIS\\s*\\(", Pattern.CASE_INSENSITIVE); @@ -142,12 +153,14 @@ public class SQLFunctionUtil { public static final Pattern strFormatDateTime = Pattern.compile("\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}", Pattern.CASE_INSENSITIVE); //other + public static final Pattern pRollUp = Pattern.compile("\\Wrollup\\s*\\(", Pattern.CASE_INSENSITIVE); public static final Pattern pQuantile = Pattern.compile("\\WQUANTILE\\s*\\("); public static final Pattern pMedian = Pattern.compile("\\WMEDIAN\\s*\\("); public static final Pattern pCast = Pattern.compile("\\WCAST\\s*\\("); public static final Pattern pNotEmpty = Pattern.compile("\\b" + NOT_EMPTY + "\\("); public static final Pattern pEmpty = Pattern.compile("\\b" + EMPTY + "\\("); public static final Pattern pConcat = Pattern.compile("\\b" + CONCAT + "\\((?!toString\\(|CAST\\()", Pattern.CASE_INSENSITIVE); + public static final Pattern pIsIPAddressInRange = Pattern.compile("\\WisIPAddressInRange\\s*\\(", Pattern.CASE_INSENSITIVE); /** * 解析mysql函数替换为对应数据库类型函数 @@ -159,6 +172,7 @@ public class SQLFunctionUtil { public static String generateDateFunction(String sql, String dbType) { sql = translateCommonFun(sql, dbType); sql = parseQuantile(sql, dbType); + sql = parseIsIPAddressInRange(sql, dbType); sql = parseMedian(sql, dbType); sql = parseCast(sql, dbType); sql = parseMergepFunction(sql, dbType); @@ -179,6 +193,44 @@ public class SQLFunctionUtil { return sql; } + public static String replaceRollup(String sql) { + sql = parseRollup(sql); + return sql; + } + + public static String parseRollup(String sql) { + while (true) { + Matcher matcher = pRollUp.matcher(sql); + if (!matcher.find()) { + return sql; + } + int start = matcher.start() + 1; + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = translateRollup(sqlParse, bracketsMatch[0], null); + sql = sql.substring(0, start) + str; + } + } + + public static String translateRollup(String sqlParse, int num, String dbType) { + Pattern pDateFormatMark = Pattern.compile("(ROLLUP)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); + Matcher mDateFormatMark = pDateFormatMark.matcher(sqlParse); + StringBuffer sb = new StringBuffer(); + if (mDateFormatMark.find()) { + String group2 = mDateFormatMark.group(2); + String replaceValue = group2; + if (StringUtil.getBracketsMatch(group2, "(", false)[0] >= 0) { + replaceValue = mDateFormatMark.group(2); + } + mDateFormatMark.appendReplacement(sb, replaceValue); + } + mDateFormatMark.appendTail(sb); + return sb.toString(); + } + public static String timeZoneFunc(String sql, String dbType) { while (true) { Matcher matcher = pTimeZone.matcher(sql); @@ -226,15 +278,15 @@ public class SQLFunctionUtil { int startTZInt = (Integer.parseInt(TimeZoneUtil.timeZoneTransition(startTZString))) / 100; timeDifference = startTZInt - oldTZInt; } - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { replaceValue = "addHours#[toDateTime#[" + params.get(0) + "," + params.get(1) + "#]," + timeDifference + "," + params.get(1) + "#]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { if (!strFormatDateTime.matcher(params.get(0)).find()) { replaceValue = "TIME_FORMAT#[TIME_SHIFT#[" + params.get(0) + "," + "'PT1H'," + timeDifference + "," + params.get(1) + "#],'yyyy-MM-dd HH:mm:ss'#]"; } else { replaceValue = "TIME_FORMAT#[TIME_SHIFT#[TIME_PARSE#[" + params.get(0) + "#]," + "'PT1H'," + timeDifference + "," + params.get(1) + "#],'yyyy-MM-dd HH:mm:ss'#]"; } - } else if (DBTypeEnum.HBASE.getValue().equals(dbType)) { + } else if (DBEngineType.HBASE.getValue().equals(dbType)) { if (!strFormatDateTime.matcher(params.get(0)).find()) { replaceValue = "TO_CHAR#[" + func + "#[ " + params.get(0) + ",'" + TimeZoneUtil.timeZoneTransition(oldTZString) + "','" + TimeZoneUtil.timeZoneTransition(startTZString) + "'#],'yyyy-MM-dd HH:mm:ss'#]"; } else { @@ -248,6 +300,7 @@ public class SQLFunctionUtil { public static String getDistributedGroupByFunction(String sourceAggregateFunctionName) { return distributedGroupByFunction.get(sourceAggregateFunctionName); } + /** * 公共函数转义 * @@ -289,6 +342,7 @@ public class SQLFunctionUtil { private static String translateUDF(String dbType, String resultSql) { resultSql = translateByPattern(resultSql, dbType, pTimeFloorWithFill); resultSql = translateByPattern(resultSql, dbType, pMaxDuration); + resultSql = translateByPattern(resultSql, dbType, pRate); resultSql = translateByPattern(resultSql, dbType, pIpToGeo); resultSql = translateByPattern(resultSql, dbType, pIpToCity); resultSql = translateByPattern(resultSql, dbType, pIpToCountry); @@ -312,7 +366,7 @@ public class SQLFunctionUtil { * @return */ public static String getQueryValue(String query, String dbType) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { query = generateDateFunction("select " + query + " as actualValue", dbType); query = Encodes.urlEncode(query); Map<String, String> results = executeHttpGetOfCK(query); @@ -321,7 +375,7 @@ public class SQLFunctionUtil { Map<String, Object> data = dates.get(0); Object actualValue = data.get("actualValue"); return actualValue.toString(); - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { query = generateDateFunction("select TIME_FORMAT(" + query + ", 'yyyy-MM-dd HH:mm:ss') as actualValue", dbType); Map<String, String> results = executeHttpPostOfDruid(query); List<Object> maps = (List<Object>) JSON.parseObject(results.get("result"), Object.class); @@ -446,6 +500,30 @@ public class SQLFunctionUtil { } /** + * 匹配范围：isIPAddressInRange( + * + * @param sql 需要解析的sql + * @param dbType 数据库的类型 + * @return + */ + public static String parseIsIPAddressInRange(String sql, String dbType) { + while (true) { + Matcher matcher = pIsIPAddressInRange.matcher(sql); + if (!matcher.find()) { + return sql; + } + int start = matcher.start() + 1; + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = parseIsIPAddressInRangeReplace(sqlParse, bracketsMatch[0], dbType); + sql = sql.substring(0, start) + str; + } + } + + /** * 替换：QUANTILE(expr,level)、QUANTILE(expr) * * @param sqlParse @@ -470,6 +548,30 @@ public class SQLFunctionUtil { } /** + * 替换：isIPAddressInRange + * + * @param sqlParse + * @param num 包含括号的个数 + * @param dbType + * @return + */ + public static String parseIsIPAddressInRangeReplace(String sqlParse, int num, String dbType) { + Pattern pNumQuantile = Pattern.compile("(isIPAddressInRange)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)"); + Matcher matcher = pNumQuantile.matcher(sqlParse); + StringBuffer sb = new StringBuffer(); + if (matcher.find()) { + String group2 = matcher.group(2); + String replaceValue = group2; + if (StringUtil.getBracketsMatch(group2, "(", false)[0] >= 0) { + replaceValue = isIPAddressInRangeReplaceByDB(group2, dbType); + } + matcher.appendReplacement(sb, replaceValue); + } + matcher.appendTail(sb); + return sb.toString(); + } + + /** * 替换：匹配范围：QUANTILE( * * @param param 截取到的括号内的参数 @@ -482,18 +584,34 @@ public class SQLFunctionUtil { if (params.size() == 1) { params.add("0.5"); } - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { replaceValue = "quantile#[" + params.get(1) + "#]#[" + params.get(0) + "#]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { replaceValue = "APPROX_QUANTILE_DS#[" + params.get(0) + "," + params.get(1) + "#]"; } return replaceValue; } - public static String parseMedian(String sql, String dbType){ + /** + * 替换：匹配范围：isIPAddressInRange( + * + * @param param 截取到的括号内的参数 + * @param dbType + * @return + */ + private static String isIPAddressInRangeReplaceByDB(String param, String dbType) { + List<String> params = diviParam(param, ","); + String replaceValue = null; + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { + replaceValue = "IF#[empty#[" + params.get(0) + "#], 0, isIPAddressInRange#[" + params.get(0) + "," + params.get(1) + "#]#]"; + } + return replaceValue; + } + + public static String parseMedian(String sql, String dbType) { while (true) { Matcher matcher = pMedian.matcher(sql); - if (!matcher.find()){ + if (!matcher.find()) { return sql; } int start = matcher.start() + 1; @@ -507,7 +625,7 @@ public class SQLFunctionUtil { } } - public static String parseMedianReplace(String sqlParse, int num, String dbType){ + public static String parseMedianReplace(String sqlParse, int num, String dbType) { Pattern pNumMedian = Pattern.compile("(MEDIAN)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)"); Matcher matcher = pNumMedian.matcher(sqlParse); StringBuffer sb = new StringBuffer(); @@ -524,11 +642,11 @@ public class SQLFunctionUtil { } - private static String medianReplaceByDB(String param, String dbType){ + private static String medianReplaceByDB(String param, String dbType) { String replaceValue = null; - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { replaceValue = "median#[" + param + "#]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { replaceValue = "APPROX_QUANTILE_DS#[" + param + "," + 0.5 + "#]"; } return replaceValue; @@ -574,15 +692,15 @@ public class SQLFunctionUtil { Expression expression = CCJSqlParserUtil.parseExpression(param); if (expression instanceof CastExpression) { CastExpression castExpression = (CastExpression) expression; - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { String dataType = castExpression.getType().getDataType(); castExpression.getType().setDataType("Nullable(" + dataType + ")"); } replaceValue = String.valueOf(castExpression).replace("(", "#[").replace(")", "#]"); } } catch (JSQLParserException e) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); } return replaceValue; } @@ -679,12 +797,13 @@ public class SQLFunctionUtil { /** * Desc 匹配UNIX_TIMESTAMP_MILLIS( + * * @param sql * @param dbType * @return */ - public static String parseUnixTimeMillis(String sql, String dbType){ + public static String parseUnixTimeMillis(String sql, String dbType) { while (true) { Matcher matcher = pUnixTimeMillis.matcher(sql); if (!matcher.find()) { @@ -853,16 +972,16 @@ public class SQLFunctionUtil { String var = group.substring(bracketsMatch[1] + 1); var = var.replaceAll(" ", ""); String replaceValue = m.group(0); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { switch (var) { case "/60)*60),'%Y-%m-%d%H:%i:%s'": - replaceValue = "toStartOfMinute#[toDateTime#[" + param + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfMinute#[toDateTime#[" + param + ", 'UTC'#]#]" : "toStartOfMinute#[toDateTime#[" + param + "#]#]"; break; case "/3600)*3600),'%Y-%m-%d%H:%i:%s'": - replaceValue = "toStartOfHour#[toDateTime#[" + param + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfHour#[toDateTime#[" + param + ", 'UTC'#]#]" : "toStartOfHour#[toDateTime#[" + param + "#]#]"; break; } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { if (param.matches("'\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}'")) { param = "TIMESTAMP " + param; } @@ -932,13 +1051,13 @@ public class SQLFunctionUtil { if (StringUtil.getBracketsMatch(group0.substring(0, group0.length() - 1), "(", true)[0] == num) { String[] split = group4.split(","); String replaceValue = ""; - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { if (split.length == 1) { - replaceValue = "toStartOfDay#[toDateTime#[" + group2 + "#]#]+#[" + group4 + "#]*86400 )"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfDay#[toDateTime#[" + group2 + ", 'UTC'#]#]+#[" + group4 + "#]*86400 )" : "toStartOfDay#[toDateTime#[" + group2 + "#]#]+#[" + group4 + "#]*86400 )"; } else if (split.length == 2) { - replaceValue = "toStartOfDay#[toDateTime#[" + group2 + "#]#]+#[" + split[0] + "#]*86400 ," + split[1] + ")"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfDay#[toDateTime#[" + group2 + ", 'UTC'#]#]+#[" + split[0] + "#]*86400 ," + split[1] + ")" : "toStartOfDay#[toDateTime#[" + group2 + "#]#]+#[" + split[0] + "#]*86400 ," + split[1] + ")"; } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { if (split.length == 1) { replaceValue = "TIME_SHIFT#[FLOOR#[ TIMESTAMP " + group2 + " to day#],'P1D'," + group4 + "#])"; } else if (split.length == 2) { @@ -984,9 +1103,9 @@ public class SQLFunctionUtil { private static String lastDayReplace(String unit, String param, String dbType) { String replaceValue = null; if ("LAST_DAY".equals(StringUtils.upperCase(unit))) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - replaceValue = "addDays#[addMonths#[toStartOfMonth#[toDateTime#[" + param + "#]#],1#],-1#]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "addDays#[addMonths#[toStartOfMonth#[toDateTime#[" + param + ", 'UTC'#]#],1#],-1#]" : "addDays#[addMonths#[toStartOfMonth#[toDateTime#[" + param + "#]#],1#],-1#]"; + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { replaceValue = "TIME_SHIFT#[FLOOR#[TIME_SHIFT#[ TIMESTAMP " + param + ", 'P1M', 1#] to month#], 'P1D', -1#]"; } } @@ -1037,17 +1156,21 @@ public class SQLFunctionUtil { if (mYear.find()) { String group1 = mYear.group(1); String group5 = mYear.group(5); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { if (group1 != null && !"".equals(group1.trim()) && group5 != null && !"".equals(group5.trim())) { - replaceValue = "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "#]#]," + group1 + " " + group5 + "#]," + params.get(1) + " -1 #]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + ", 'UTC'#]#]," + group1 + " " + group5 + "#]," + params.get(1) + " -1 #]#]" : + "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "#]#]," + group1 + " " + group5 + "#]," + params.get(1) + " -1 #]#]"; } else if (group5 != null && !"".equals(group5.trim())) { - replaceValue = "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "#]#]," + group5 + "#]," + params.get(1) + " -1 #]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + ", 'UTC'#]#]," + group5 + "#]," + params.get(1) + " -1 #]#]" + : "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "#]#]," + group5 + "#]," + params.get(1) + " -1 #]#]"; } else if (group1 != null && !"".equals(group1.trim())) { - replaceValue = "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "#]#]," + group1 + "0 #]," + params.get(1) + " -1 #]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + ", 'UTC'#]#]," + group1 + "0 #]," + params.get(1) + " -1 #]#]" + : "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "#]#]," + group1 + "0 #]," + params.get(1) + " -1 #]#]"; } else { - replaceValue = "toDateTime#[addDays#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "#]#]," + params.get(1) + " -1 #]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[addDays#[toStartOfYear#[toDateTime#[" + mYear.group(3) + ", 'UTC'#]#]," + params.get(1) + " -1 #]#]" + : "toDateTime#[addDays#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "#]#]," + params.get(1) + " -1 #]#]"; } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { if (group1 != null && !"".equals(group1.trim()) && group5 != null && !"".equals(group5.trim())) { replaceValue = "TIME_SHIFT#[TIME_SHIFT#[FLOOR#[ TIMESTAMP " + mYear.group(3) + " to year#],'P1Y'," + group1 + " " + group5 + " #],'P1D'," + params.get(1) + " -1 #]"; } else if (group5 != null && !"".equals(group5.trim())) { @@ -1095,42 +1218,42 @@ public class SQLFunctionUtil { private static String unitFuncGetNumReplaceByDB(String unit, String expr, String dbType) { String replaceValue = null; unit = StringUtils.upperCase(unit); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { switch (unit) { case "YEAR": - replaceValue = "toYear#[toDateTime#[" + expr + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toYear#[toDateTime#[" + expr + ", 'UTC'#]#]" : "toYear#[toDateTime#[" + expr + "#]#]"; break; case "QUARTER": - replaceValue = "toQuarter#[toDateTime#[" + expr + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toQuarter#[toDateTime#[" + expr + ", 'UTC'#]#]" : "toQuarter#[toDateTime#[" + expr + "#]#]"; break; case "MONTH": - replaceValue = "toMonth#[toDateTime#[" + expr + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toMonth#[toDateTime#[" + expr + ", 'UTC'#]#]" : "toMonth#[toDateTime#[" + expr + "#]#]"; break; case "DAY": - replaceValue = "toDayOfMonth#[toDateTime#[" + expr + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDayOfMonth#[toDateTime#[" + expr + ", 'UTC'#]#]" : "toDayOfMonth#[toDateTime#[" + expr + "#]#]"; break; case "HOUR": - replaceValue = "toHour#[toDateTime#[" + expr + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toHour#[toDateTime#[" + expr + ", 'UTC'#]#]" : "toHour#[toDateTime#[" + expr + "#]#]"; break; case "MINUTE": - replaceValue = "toMinute#[toDateTime#[" + expr + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toMinute#[toDateTime#[" + expr + ", 'UTC'#]#]" : "toMinute#[toDateTime#[" + expr + "#]#]"; break; case "SECOND": - replaceValue = "toSecond#[toDateTime#[" + expr + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toSecond#[toDateTime#[" + expr + ", 'UTC'#]#]" : "toSecond#[toDateTime#[" + expr + "#]#]"; break; case "DAYOFYEAR": - replaceValue = "toDayOfYear#[toDateTime#[" + expr + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDayOfYear#[toDateTime#[" + expr + ", 'UTC'#]#]" : "toDayOfYear#[toDateTime#[" + expr + "#]#]"; break; case "DAYOFMONTH": - replaceValue = "toDayOfMonth#[toDateTime#[" + expr + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDayOfMonth#[toDateTime#[" + expr + ", 'UTC'#]#]" : "toDayOfMonth#[toDateTime#[" + expr + "#]#]"; break; case "DAYOFWEEK": - replaceValue = "toDayOfWeek#[addDays#[toDateTime#[" + expr + "#],1#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDayOfWeek#[addDays#[toDateTime#[" + expr + ", 'UTC'#],1#]#]" : "toDayOfWeek#[addDays#[toDateTime#[" + expr + "#],1#]#]"; break; default: replaceValue = unit + "#[" + expr + "#]"; } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { switch (unit) { case "YEAR": replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0#], 'YEAR' #]"; @@ -1188,37 +1311,37 @@ public class SQLFunctionUtil { if (!sign) { param2 = "- #[" + param2 + "#]"; } - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { switch (StringUtils.upperCase(matcher.group(2))) { case "SECOND": - replaceValue = "toDateTime#[" + param1 + "#]" + param2; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[" + param1 + ", 'UTC'#]" + param2 : "toDateTime#[" + param1 + "#]" + param2; break; case "MINUTE": - replaceValue = "toDateTime#[" + param1 + "#]" + param2 + "*60"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[" + param1 + ", 'UTC'#]" + param2 + "*60" : "toDateTime#[" + param1 + "#]" + param2 + "*60"; break; case "HOUR": - replaceValue = "toDateTime#[" + param1 + "#]" + param2 + "*3600"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[" + param1 + ", 'UTC'#]" + param2 + "*3600" : "toDateTime#[" + param1 + "#]" + param2 + "*3600"; break; case "DAY": - replaceValue = "toDateTime#[" + param1 + "#]" + param2 + "*86400"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[" + param1 + ", 'UTC'#]" + param2 + "*86400" : "toDateTime#[" + param1 + "#]" + param2 + "*86400"; break; case "WEEK": - replaceValue = "toDateTime#[" + param1 + "#]" + param2 + "*604800"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[" + param1 + ", 'UTC'#]" + param2 + "*604800" : "toDateTime#[" + param1 + "#]" + param2 + "*604800"; break; case "MONTH": - replaceValue = "addMonths#[toDateTime#[" + param1 + "#] ," + param2 + "#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "addMonths#[toDateTime#[" + param1 + ", 'UTC'#] ," + param2 + "#]" : "addMonths#[toDateTime#[" + param1 + "#] ," + param2 + "#]"; break; case "QUARTER": - replaceValue = "addQuarter#[toDateTime#[" + param1 + "#]," + param2 + "#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "addQuarter#[toDateTime#[" + param1 + ", 'UTC'#]," + param2 + "#]" : "addQuarter#[toDateTime#[" + param1 + "#]," + param2 + "#]"; break; case "YEAR": - replaceValue = "addYears#[toDateTime#[" + param1 + "#]," + param2 + "#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "addYears#[toDateTime#[" + param1 + ", 'UTC'#]," + param2 + "#]" : "addYears#[toDateTime#[" + param1 + "#]," + param2 + "#]"; break; default: replaceValue = param1 + param2; break; } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { switch (StringUtils.upperCase(matcher.group(2))) { case "SECOND": replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'PT1S'," + param2 + "#]"; @@ -1257,12 +1380,12 @@ public class SQLFunctionUtil { if (!sign) { param2 = "- #[" + param2 + "#]"; } - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - replaceValue = "addDays#[toDateTime#[" + param1 + "#]," + param2 + "#]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "addDays#[toDateTime#[" + param1 + ", 'UTC'#]," + param2 + "#]" : "addDays#[toDateTime#[" + param1 + "#]," + param2 + "#]"; + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1D'," + param2 + "#]"; } - } else if (params.size() == 3 && DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + } else if (params.size() == 3 && DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { replaceValue = "DATE_ADD#[" + params.get(0) + "," + params.get(1) + "," + params.get(2) + "#]"; } @@ -1282,14 +1405,14 @@ public class SQLFunctionUtil { String param1 = params.get(0); String replaceValue = null; if ("FROM_UNIXTIME".equals(StringUtils.upperCase(func)) && params.size() == 1) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - replaceValue = "toDateTime#[" + param + "#]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && !bool) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[" + param + ", 'UTC'#]" : "toDateTime#[" + param + "#]"; + } else if (DBEngineType.DRUID.getValue().equals(dbType) && !bool) { replaceValue = "MILLIS_TO_TIMESTAMP#[ 1000 * #[" + param + "#]#]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && bool) { + } else if (DBEngineType.DRUID.getValue().equals(dbType) && bool) { replaceValue = "TIME_FORMAT#[MILLIS_TO_TIMESTAMP#[ 1000 * #[" + param + "#]#],'YYYY-MM-dd HH:mm:ss'#]"; } - } else if (func.contains("FROM_UNIXTIME") && DBTypeEnum.DRUID.getValue().equals(dbType) && "%Y-%m-%d %H:%i:%s".equals(params.get(1).replaceAll("\\'|\\\"", "").trim())) { + } else if (func.contains("FROM_UNIXTIME") && DBEngineType.DRUID.getValue().equals(dbType) && "%Y-%m-%d %H:%i:%s".equals(params.get(1).replaceAll("\\'|\\\"", "").trim())) { if (!bool) { replaceValue = "MILLIS_TO_TIMESTAMP#[ 1000*#[" + param1 + "#]#]"; } else if (bool) { @@ -1297,71 +1420,71 @@ public class SQLFunctionUtil { } } else { String param2 = params.get(1).replaceAll("\\'|\\\"", "").trim(); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType) && bool) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType) && bool) { switch (param2) { case "%Y-%m-%d %H:%i:%s": - replaceValue = "toDateTime#[" + param1 + "#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[" + param1 + ", 'UTC'#]" : "toDateTime#[" + param1 + "#]"; break; case "%Y-%m-%d %H:%i:00": - replaceValue = "toStartOfMinute#[toDateTime#[" + param1 + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfMinute#[toDateTime#[" + param1 + ", 'UTC'#]#]" : "toStartOfMinute#[toDateTime#[" + param1 + "#]#]"; break; case "%Y-%m-%d %H:00:00": - replaceValue = "toStartOfHour#[toDateTime#[" + param1 + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfHour#[toDateTime#[" + param1 + ", 'UTC'#]#]" : "toStartOfHour#[toDateTime#[" + param1 + "#]#]"; break; case "%Y-%m-%d 00:00:00": - replaceValue = "toStartOfDay#[toDateTime#[" + param1 + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfDay#[toDateTime#[" + param1 + ", 'UTC'#]#]" : "toStartOfDay#[toDateTime#[" + param1 + "#]#]"; break; case "%Y-%m-01 00:00:00": - replaceValue = "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + "#]#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + ", 'UTC'#]#]#]" : "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + "#]#]#]"; break; case "%Y-01-01 00:00:00": - replaceValue = "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + "#]#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + ", 'UTC'#]#]#]" : "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + "#]#]#]"; break; case "%Y-%m-%d": - replaceValue = "formatDateTime#[toDateTime#[" + param1 + "#], '%Y-%m-%d'#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "formatDateTime#[toDateTime#[" + param1 + ", 'UTC'#], '%Y-%m-%d'#]" : "formatDateTime#[toDateTime#[" + param1 + "#], '%Y-%m-%d'#]"; break; case "%Y-%m-01": - replaceValue = "formatDateTime#[toDateTime#[" + param1 + "#], '%Y-%m-01'#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "formatDateTime#[toDateTime#[" + param1 + ", 'UTC'#], '%Y-%m-01'#]" : "formatDateTime#[toDateTime#[" + param1 + "#], '%Y-%m-01'#]"; break; case "%Y-01-01": - replaceValue = "formatDateTime#[toDateTime#[" + param1 + "#], '%Y-01-01'#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "formatDateTime#[toDateTime#[" + param1 + ", 'UTC'#], '%Y-01-01'#]" : "formatDateTime#[toDateTime#[" + param1 + "#], '%Y-01-01'#]"; break; default: replaceValue = "toDateTime#[" + param1 + "#]"; } - } else if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType) && !bool) { + } else if (DBEngineType.CLICKHOUSE.getValue().equals(dbType) && !bool) { switch (param2) { case "%Y-%m-%d %H:%i:%s": - replaceValue = "toDateTime#[" + param1 + "#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[" + param1 + ", 'UTC'#]" : "toDateTime#[" + param1 + "#]"; break; case "%Y-%m-%d %H:%i:00": - replaceValue = "toStartOfMinute#[toDateTime#[" + param1 + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfMinute#[toDateTime#[" + param1 + ", 'UTC'#]#]" : "toStartOfMinute#[toDateTime#[" + param1 + "#]#]"; break; case "%Y-%m-%d %H:00:00": - replaceValue = "toStartOfHour#[toDateTime#[" + param1 + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfHour#[toDateTime#[" + param1 + ", 'UTC'#]#]" : "toStartOfHour#[toDateTime#[" + param1 + "#]#]"; break; case "%Y-%m-%d 00:00:00": - replaceValue = "toStartOfDay#[toDateTime#[" + param1 + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfDay#[toDateTime#[" + param1 + ", 'UTC'#]#]" : "toStartOfDay#[toDateTime#[" + param1 + "#]#]"; break; case "%Y-%m-01 00:00:00": - replaceValue = "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + "#]#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + ", 'UTC'#]#]#]" : "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + "#]#]#]"; break; case "%Y-01-01 00:00:00": - replaceValue = "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + "#]#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + ", 'UTC'#]#]#]" : "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + "#]#]#]"; break; case "%Y-%m-%d": - replaceValue = "toStartOfDay#[toDateTime#[" + param1 + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfDay#[toDateTime#[" + param1 + ", 'UTC'#]#]" : "toStartOfDay#[toDateTime#[" + param1 + "#]#]"; break; case "%Y-%m-01": - replaceValue = "toStartOfMonth#[toDateTime#[" + param1 + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfMonth#[toDateTime#[" + param1 + ", 'UTC'#]#]" : "toStartOfMonth#[toDateTime#[" + param1 + "#]#]"; break; case "%Y-01-01": - replaceValue = "toStartOfYear#[toDateTime#[" + param1 + "#]#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfYear#[toDateTime#[" + param1 + ", 'UTC'#]#]" : "toStartOfYear#[toDateTime#[" + param1 + "#]#]"; break; default: - replaceValue = "toDateTime#[" + param1 + "#]"; + replaceValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toDateTime#[" + param1 + ", 'UTC'#]" : "toDateTime#[" + param1 + "#]"; } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && bool) { + } else if (DBEngineType.DRUID.getValue().equals(dbType) && bool) { switch (param2) { case "%Y-%m-%d %H:%i:%s": replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:mm:ss'#]"; @@ -1393,7 +1516,7 @@ public class SQLFunctionUtil { default: replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:mm:ss'#]"; } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && !bool) { + } else if (DBEngineType.DRUID.getValue().equals(dbType) && !bool) { switch (param2) { case "%Y-%m-%d %H:%i:%s": replaceValue = " TIMESTAMP " + param1; @@ -1443,7 +1566,7 @@ public class SQLFunctionUtil { return sql; } StringBuffer sb = new StringBuffer(); - if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (DBEngineType.DRUID.getValue().equals(dbType)) { Matcher mNowFun = pNow.matcher(sql); while (mNowFun.find()) { @@ -1529,23 +1652,23 @@ public class SQLFunctionUtil { } - public static String parseFromUnixTimeMillisByDB(String sqlParse, int num, String dbType){ + public static String parseFromUnixTimeMillisByDB(String sqlParse, int num, String dbType) { Pattern pFromUnixTimeMillisParse = Pattern.compile("(FROM_UNIXTIME_MILLIS)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); Matcher matcherParse = pFromUnixTimeMillisParse.matcher(sqlParse); StringBuffer sb = new StringBuffer(); String innerValue = null; if (matcherParse.find()) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { innerValue = "fromUnixTimestamp64Milli#[CAST#[toUnixTimestamp64Milli#[CAST#[now#[#] as DateTime64#]#], 'Int64'#]#]"; } else { innerValue = String.format("fromUnixTimestamp64Milli#[CAST#[%s, 'Int64'#]#]", matcherParse.group(2)); } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { innerValue = "TIME_FORMAT#[MILLIS_TO_TIMESTAMP#[TIMESTAMP_TO_MILLIS#[CURRENT_TIMESTAMP#]#],'yyyy-MM-dd HH:mm:ss.SSS'#]"; } else { - innerValue = String.format("TIME_FORMAT#[MILLIS_TO_TIMESTAMP#[%s#],'yyyy-MM-dd HH:mm:ss.SSS'#]",matcherParse.group(2)); + innerValue = String.format("TIME_FORMAT#[MILLIS_TO_TIMESTAMP#[%s#],'yyyy-MM-dd HH:mm:ss.SSS'#]", matcherParse.group(2)); } } matcherParse.appendReplacement(sb, innerValue); @@ -1568,20 +1691,20 @@ public class SQLFunctionUtil { StringBuffer sb = new StringBuffer(); String innerValue = null; if (matcherParse.find()) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { innerValue = "toUnixTimestamp#[now#[#]#]"; } else { - innerValue = "toUnixTimestamp#[parseDateTimeBestEffort#[toString#[ " + matcherParse.group(2) + "#]#]#]"; + innerValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toUnixTimestamp#[parseDateTimeBestEffort#[toString#[ " + matcherParse.group(2) + "#], 'UTC'#]#]" : "toUnixTimestamp#[parseDateTimeBestEffort#[toString#[ " + matcherParse.group(2) + "#]#]#]"; } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { - innerValue = "0.001 * TIMESTAMP_TO_MILLIS#[CURRENT_TIMESTAMP#]"; + innerValue = "TIMESTAMP_TO_MILLIS#[CURRENT_TIMESTAMP#]/1000"; } else { if (matcherParse.group(2).startsWith("'") && matcherParse.group(2).endsWith("'")) { - innerValue = "TIMESTAMP_TO_MILLIS#[ TIME_PARSE#[TIME_FORMAT#[ TIMESTAMP " + matcherParse.group(2) + ", 'yyyy-MM-dd HH:mm:ss'#]#]#]/1000"; + innerValue = "TIMESTAMP_TO_MILLIS#[ TIME_PARSE#[ " + matcherParse.group(2) + "#]#]/1000"; } else { - innerValue = "0.001 * TIMESTAMP_TO_MILLIS#[ TIMESTAMP " + matcherParse.group(2) + "#]"; + innerValue = "TIMESTAMP_TO_MILLIS#[ TIMESTAMP " + matcherParse.group(2) + "#]/1000"; } } } @@ -1591,24 +1714,24 @@ public class SQLFunctionUtil { return sb.toString(); } - public static String parseUnixTimeMillisReplaceByDB(String sqlParse, int num, String dbType){ + public static String parseUnixTimeMillisReplaceByDB(String sqlParse, int num, String dbType) { Pattern pUnixTimeMillisParse = Pattern.compile("(UNIX_TIMESTAMP_MILLIS)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); Matcher matcherParse = pUnixTimeMillisParse.matcher(sqlParse); StringBuffer sb = new StringBuffer(); String innerValue = null; if (matcherParse.find()) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { innerValue = "toUnixTimestamp64Milli#[CAST#[now#[#] as DateTime64#]#]"; } else { - innerValue = "toUnixTimestamp64Milli#[CAST#[parseDateTime64BestEffort#[toString#[ " + matcherParse.group(2) + "#]#] as DateTime64#]#]"; + innerValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toUnixTimestamp64Milli#[CAST#[parseDateTime64BestEffort#[toString#[ " + matcherParse.group(2) + "#], 'UTC'#] as DateTime64#]#]" : "toUnixTimestamp64Milli#[CAST#[parseDateTime64BestEffort#[toString#[ " + matcherParse.group(2) + "#]#] as DateTime64#]#]"; } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { innerValue = "TIMESTAMP_TO_MILLIS#[CURRENT_TIMESTAMP#]"; } else { if (matcherParse.group(2).startsWith("'") && matcherParse.group(2).endsWith("'")) { - innerValue = String.format("TIMESTAMP_TO_MILLIS#[ TIME_PARSE#[%s#]#]",matcherParse.group(2)); + innerValue = String.format("TIMESTAMP_TO_MILLIS#[ TIME_PARSE#[%s#]#]", matcherParse.group(2)); } else { innerValue = "TIMESTAMP_TO_MILLIS#[ " + matcherParse.group(2) + "#]"; } @@ -1634,9 +1757,10 @@ public class SQLFunctionUtil { StringBuffer sb = new StringBuffer(); String innerValue = null; if (matcherParse.find()) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - innerValue = "toStartOfDay#[parseDateTimeBestEffort#[toString#[" + matcherParse.group(2) + "#]#]#]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { + String param = matcherParse.group(2); + innerValue = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toStartOfDay#[parseDateTimeBestEffort#[toString#[" + param + "#], 'UTC'#]#]" : "toStartOfDay#[parseDateTimeBestEffort#[toString#[" + param + "#]#]#]"; + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { innerValue = "TIME_FLOOR#[ TIMESTAMP " + matcherParse.group(2) + ",'P1D'#]"; } matcherParse.appendReplacement(sb, innerValue); @@ -1652,10 +1776,7 @@ public class SQLFunctionUtil { * @param div 参数分隔符 * @return */ - public static List<String> diviParam(String str, String div) { - if (str == null) { - return null; - } + public static List<String> diviParam(@NotNull String str, String div) { List<String> result = new ArrayList<>(); String[] split = str.split(div); String resultTemp = ""; @@ -1731,13 +1852,16 @@ public class SQLFunctionUtil { private static String translateCaseInsensitive(Function fun, String dbType) { String data; - switch (fun.getName()){ + switch (fun.getName()) { case TIME_FLOOR_WITH_FILL: data = translateTimeFloorWithFill(fun, dbType); break; case MAX_DURATION: data = translateDuration(fun, dbType); break; + case RATE: + data = translateRate(fun, dbType); + break; case NOT_EMPTY: data = translateNotEmpty(fun, dbType); break; @@ -1767,6 +1891,7 @@ public class SQLFunctionUtil { } return data; } + /** * Desc: 转义notEmpty * @@ -1778,7 +1903,7 @@ public class SQLFunctionUtil { */ private static String translateNotEmpty(Function fun, String dbType) { String data; - if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (DBEngineType.DRUID.getValue().equals(dbType)) { String expr = fun.getParamsList().get(0); data = expr + " IS NOT NUll "; } else { @@ -1804,7 +1929,7 @@ public class SQLFunctionUtil { */ private static String translateEmpty(Function fun, String dbType) { String data; - if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (DBEngineType.DRUID.getValue().equals(dbType)) { String expr = fun.getParamsList().get(0); data = expr + " IS NUll "; } else { @@ -1839,17 +1964,17 @@ public class SQLFunctionUtil { if (matcherPT.find()) { String num = matcherPT.group(1); String unit = matcherPT.group(2); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { String format = null; if ("S".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s SECOND)))"; + format = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s, 'UTC'),INTERVAL %s SECOND)))" : "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s SECOND)))"; } else if ("M".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s MINUTE)))"; + format = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s, 'UTC'),INTERVAL %s MINUTE)))" : "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s MINUTE)))"; } else if ("H".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s HOUR)))"; + format = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s, 'UTC'),INTERVAL %s HOUR)))" : "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s HOUR)))"; } return String.format(format, timestamp, num); - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { String format = "TIMESTAMP_TO_MILLIS(time_floor(MILLIS_TO_TIMESTAMP(%s * 1000),%s))/1000"; return String.format(format, timestamp, period); } @@ -1859,19 +1984,19 @@ public class SQLFunctionUtil { if (matcherP.find()) { String num = matcherP.group(1); String unit = matcherP.group(2); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { String format = null; if ("D".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s DAY)))"; + format = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s, 'UTC'),INTERVAL %s DAY)))" : "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s DAY)))"; } else if ("W".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s WEEK)))"; + format = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s, 'UTC'),INTERVAL %s WEEK)))" : "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s WEEK)))"; } else if ("M".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s MONTH)))"; + format = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s, 'UTC'),INTERVAL %s MONTH)))" : "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s MONTH)))"; } else if ("Y".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s YEAR)))"; + format = CONVERT_DEFAULT_TIMEZONE_INTO_UTC ? "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s, 'UTC'),INTERVAL %s YEAR)))" : "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s YEAR)))"; } return String.format(format, timestamp, num); - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbType)) { String format = "TIMESTAMP_TO_MILLIS(time_floor(MILLIS_TO_TIMESTAMP(%s * 1000),%s))/1000"; return String.format(format, timestamp, period); } @@ -1892,7 +2017,7 @@ public class SQLFunctionUtil { int factor = 60; String timestampExpr = fun.getParamsList().get(0); int interval = Integer.parseInt(fun.getParamsList().get(1)); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { return String.format("length(arrayReduce('max', alphaTokens(replaceAll(replaceAll(toString(arrayMap(x-> x<(%s+%s), arrayDifference(arraySort(groupUniqArray(%s))))),'1','a'),',','') ) )) * %s " , interval, factor, timestampExpr, interval); } @@ -1900,6 +2025,37 @@ public class SQLFunctionUtil { } /** + * Desc: 转义RATE + * + * @param fun + * @param dbType + * @return {@link String} + * @created by wWei + * @date 2023/12/22 4:47 下午 + */ + private static String translateRate(Function fun, String dbType) { + List<String> paramsList = fun.paramsList; + if (DBEngineType.DRUID.getValue().equals(dbType) || DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { + if (paramsList.size() == 2) { + String durationParam = fun.getParamsList().get(1); + if (!NumberUtil.isNumber(durationParam) && durationParam.startsWith("'") && durationParam.endsWith("'")) { + durationParam = String.valueOf(Period.parse(StrUtil.strip(durationParam, "'")).toStandardSeconds().getSeconds()); + } + String formatString = "ROUND(SUM(%s) * 1.0 / %s, 2)"; + return String.format(formatString, paramsList.get(0), durationParam); + } else if (paramsList.size() == 3) { + String durationParam = fun.getParamsList().get(1); + if (!NumberUtil.isNumber(durationParam)) { + durationParam = String.valueOf(Period.parse(StrUtil.strip(durationParam, "'")).toStandardSeconds().getSeconds()); + } + String formatString = "ROUND(SUM(%s) / (%s * 1.0/%s), 2)"; + return String.format(formatString, paramsList.get(0), durationParam, paramsList.get(2)); + } + } + return fun.toString(); + } + + /** * Desc: 转义自定义函数 MEDIAN_HDR/QUANTILE_HDR/PERCENTILES_HDR/APPROX_COUNT_DISTINCT_HLLD * * @param fun @@ -1909,7 +2065,7 @@ public class SQLFunctionUtil { private static String translateHdrHistogram(Function fun, String dbType) { String hdrSketchExpr = fun.getParamsList().get(0); - if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (DBEngineType.DRUID.getValue().equals(dbType)) { switch (fun.getName()) { case MEDIAN_HDR: return String.format("APPROX_QUANTILE_HDR(%s,0.50)", hdrSketchExpr); @@ -1917,7 +2073,7 @@ public class SQLFunctionUtil { double probability = Double.parseDouble(fun.getParamsList().get(1)); return String.format("APPROX_QUANTILE_HDR(%s,%s)", hdrSketchExpr, probability); case PERCENTILES_HDR: - return String.format("HDR_GET_PERCENTILES(HDR_HISTOGRAM(%s), 2)", hdrSketchExpr); + return String.format("HDR_GET_PERCENTILES(HDR_HISTOGRAM(%s), 5)", hdrSketchExpr); case APPROX_COUNT_DISTINCT_HLLD: return String.format("APPROX_COUNT_DISTINCT_HLLD#[%s#]", hdrSketchExpr); } @@ -1927,13 +2083,14 @@ public class SQLFunctionUtil { /** * Desc 转义 COUNT_DISTINCT + * * @param fun * @param dbType * @return */ private static String translateCountDistinct(Function fun, String dbType) { String field = fun.getParamsList().get(0); - if (DBTypeEnum.DRUID.getValue().equals(dbType) || DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)){ + if (DBEngineType.DRUID.getValue().equals(dbType) || DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { return String.format("count(distinct %s)", field); } return fun.toString(); @@ -1948,9 +2105,9 @@ public class SQLFunctionUtil { */ private static String translateConcat(Function function, String dbType) { StringBuffer sb = new StringBuffer(); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbType)) { for (String param : function.getParamsList()) { - sb.append("toString("+param+"), "); + sb.append("toString(" + param + "), "); } return String.format("concat(%s'')", sb); } diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/SQLHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/SQLHelper.java index dae6f3ed..7e8fe1b8 100644 --- a/src/main/java/com/mesalab/common/utils/sqlparser/SQLHelper.java +++ b/src/main/java/com/mesalab/common/utils/sqlparser/SQLHelper.java @@ -3,12 +3,11 @@ package com.mesalab.common.utils.sqlparser; import com.alibaba.druid.util.JdbcConstants; import com.geedgenetworks.utils.StringUtil; import com.google.common.collect.Lists; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.DBEngineType; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.qgw.constant.QGWMessageConst; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; +import com.mesalab.common.exception.CommonErrorCode; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -31,6 +30,8 @@ public class SQLHelper { public static final Pattern pOrder = Pattern.compile("order\\s*by[\\w|\\W|\\s|\\S]*", Pattern.CASE_INSENSITIVE); public static final String INVALID_LIMIT_DESC = "NO LIMIT"; + private static final Pattern PATTERN_QUOTES_AND_BACKTICKS = Pattern.compile("^([\"`])(.*)([\"`])$"); + /** * 去除qlString的select子句。 * @@ -58,6 +59,22 @@ public class SQLHelper { return sb.toString(); } + /** + * Desc: 去除字符串首位引号或反引号 + * + * @param str + * @return {@link String} + * @created by wWei + * @date 2024/3/11 17:31 + */ + public static String removeQuotesAndBackticks(String str) { + Matcher matcher = PATTERN_QUOTES_AND_BACKTICKS.matcher(str); + if (matcher.matches()) { + return matcher.group(2); + } + return str; + } + /** * 为当前SQL最外层增加Limit限制 @@ -89,15 +106,15 @@ public class SQLHelper { /** * 获取当前数据库预定义的方言 * - * @param dbType + * @param dbEngine * @return */ - public static String getDialectDBType(String dbType) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + public static String getDialectType(String dbEngine) { + if (DBEngineType.CLICKHOUSE.getValue().equals(dbEngine)) { return JdbcConstants.MYSQL; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + } else if (DBEngineType.DRUID.getValue().equals(dbEngine)) { return JdbcConstants.HIVE; - } else if (DBTypeEnum.HBASE.getValue().equals(dbType)) { + } else if (DBEngineType.HBASE.getValue().equals(dbEngine)) { return JdbcConstants.MYSQL; } else { return null; @@ -106,9 +123,9 @@ public class SQLHelper { } /** - * Desc: Desc: 获取当前sql的表名 + * Desc: Get Table Name from SQL * - * @param sql + * @param sql SQL * @return {@link List<String>} * @created by wWei * @date 2023/8/30 14:38 @@ -125,13 +142,65 @@ public class SQLHelper { }); } catch (JSQLParserException e) { log.error("SQL Syntax Error: Get Table Name Error, SQL is:{}, Error is:{}", sql, e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); } return result; } /** + * Desc: Get Table Name from Statement + * + * @param statement SQL statement + * @return {@link List<String>} + * @created by wWei + * @date 2023/8/30 14:38 + */ + public static List<String> getTableName(Statement statement) { + List<String> result = Lists.newArrayList(); + TablesNamesFinder tablesNamesFinder = new TablesNamesFinder(); + List<String> tableList = tablesNamesFinder.getTableList(statement); + tableList.forEach(o -> { + int i = o.lastIndexOf("."); + result.add(i < 0 ? o : o.substring(i + 1)); + }); + return result; + } + + /** + * Desc: 获取当前sql执行最大结果数 + * + * @param sql + * @return {@link Long} + * @created by wWei + * @date 2024/3/4 18:12 + */ + public static Long getRowCount(String sql) { + try { + Statement statement = CCJSqlParserUtil.parse(sql); + if (statement instanceof Select) { + Select select = (Select) statement; + SelectBody selectBody = select.getSelectBody(); + if (selectBody instanceof PlainSelect) { + PlainSelect plainSelect = (PlainSelect) selectBody; + Limit limit = plainSelect.getLimit(); + if (limit != null) { + Expression rowCount = limit.getRowCount(); + if (rowCount instanceof LongValue) { + return ((LongValue) rowCount).getValue(); + } + } + } + } + } catch (JSQLParserException e) { + log.error("SQL Syntax Error: Get Row Count Error, SQL is:{}, Error is:{}", sql, e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); + } + return null; + } + + /** * Desc: 从SQL中获取指定函数参数 * * @param sql @@ -164,13 +233,13 @@ public class SQLHelper { // selectStatement = ((ExplainStatement) statement).getStatement(); } else { log.error("Not support DML Parser"); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),QGWMessageConst.NOT_SUPPORT_DML_PARSER)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), QGWMessageConst.NOT_SUPPORT_DML_PARSER)); } } catch (JSQLParserException e) { log.error("SQL Syntax Error: Get Table Name Error, SQL is:{}, Error is:{}", sql, e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),e.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); } return Lists.newArrayList(); } @@ -197,6 +266,7 @@ public class SQLHelper { /** * Desc: 获取子查询SQL + * * @param sql * @return {@link String} * @created by wWei @@ -215,9 +285,9 @@ public class SQLHelper { plainSelect = inner; } return plainSelect; - }catch (JSQLParserException | RuntimeException e) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"statistics job init error: ".concat(e.getMessage()))); + } catch (JSQLParserException | RuntimeException e) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), "statistics job init error: ".concat(e.getMessage()))); } } diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/SQLQueryTypeHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/SQLQueryTypeHelper.java new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/sqlparser/SQLQueryTypeHelper.java diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/SQLSyntaxParserUtil.java b/src/main/java/com/mesalab/common/utils/sqlparser/SQLSyntaxParserUtil.java index b8bfc705..7c0a9642 100644 --- a/src/main/java/com/mesalab/common/utils/sqlparser/SQLSyntaxParserUtil.java +++ b/src/main/java/com/mesalab/common/utils/sqlparser/SQLSyntaxParserUtil.java @@ -5,11 +5,11 @@ import cn.hutool.log.LogFactory; import com.geedgenetworks.utils.StringUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.utils.SpringContextUtil; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.qgw.service.QueryService; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.service.SQLSyncQueryService; import lombok.Data; import lombok.NoArgsConstructor; import net.sf.jsqlparser.JSQLParserException; @@ -39,17 +39,14 @@ public class SQLSyntaxParserUtil { private static final Log log = LogFactory.get(); - private static QueryService queryService = (QueryService) SpringContextUtil.getBean("queryService"); - public static List syntaxParse(String sql) { List resultList = Lists.newArrayList(); try { parseSQL(sql, resultList); - //addTypeForSelectItems(sql, resultList); } catch (RuntimeException | JSQLParserException e) { log.error("syntax_parse: sql-" + sql + ", error: ", e); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),e.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),e.getMessage())); } return resultList; } diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/SQLVisitorUtil.java b/src/main/java/com/mesalab/common/utils/sqlparser/SQLVisitorUtil.java index 26165c6d..acf40a6c 100644 --- a/src/main/java/com/mesalab/common/utils/sqlparser/SQLVisitorUtil.java +++ b/src/main/java/com/mesalab/common/utils/sqlparser/SQLVisitorUtil.java @@ -2,7 +2,7 @@ package com.mesalab.common.utils.sqlparser; import cn.hutool.core.util.StrUtil; import com.mesalab.common.utils.SpringContextUtil; -import com.mesalab.qgw.service.MetadataService; +import com.mesalab.qgw.service.DatabaseService; import net.sf.jsqlparser.expression.*; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Table; @@ -17,7 +17,7 @@ import net.sf.jsqlparser.util.deparser.SelectDeParser; */ public class SQLVisitorUtil { - private static MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("metadataService"); + private static DatabaseService databaseService = (DatabaseService) SpringContextUtil.getBean("databaseService"); /** * Desc: 将SQL列名、别名、表名添加双引号转义 @@ -47,7 +47,7 @@ public class SQLVisitorUtil { if (!StrUtil.isBlankIfStr(name)) { table.setName(addDoubleQuote(name)); } - table.setSchemaName(addDoubleQuote(StrUtil.isBlank(table.getSchemaName()) ? metadataService.getDBNameByTableName(name) : table.getSchemaName())); + table.setSchemaName(addDoubleQuote(StrUtil.isBlank(table.getSchemaName()) ? databaseService.getDBNameByTableName(name) : table.getSchemaName())); Alias alias = table.getAlias(); if (!StrUtil.isBlankIfStr(alias)) { alias.setName(addDoubleQuote(alias.getName())); diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/DruidSampleSQLHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/SampleSQLHelper.java index 115932e5..1e5a3ff5 100644 --- a/src/main/java/com/mesalab/common/utils/sqlparser/DruidSampleSQLHelper.java +++ b/src/main/java/com/mesalab/common/utils/sqlparser/SampleSQLHelper.java @@ -5,6 +5,8 @@ import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.expression.operators.conditional.AndExpression; import net.sf.jsqlparser.expression.operators.conditional.OrExpression; import net.sf.jsqlparser.expression.operators.relational.EqualsTo; import net.sf.jsqlparser.parser.CCJSqlParserUtil; @@ -16,30 +18,33 @@ import java.util.List; /** * TODO * - * @Classname DruidSampleSQLHelper + * @Classname SampleSQLHelperOfWhereExpr * @Date 2023/3/22 10:42 * @Author wWei */ -public class DruidSampleSQLHelper { +public class SampleSQLHelper { private final String originalSQL; private final String filterExpr; - private final boolean isInvalidCondExpr; + private final Integer maxSize; + private final Boolean isInvalidCondExpr; /** * Desc: TODO * * @param originalSQL 原始SQL - * @param filterExpr 置换后的 WHERE 过滤条件 + * @param filterExpr 新增的 WHERE 过滤条件 * @param isInvalidCondExpr 是否失效 HAVING 过滤条件 + * @param defaultSize 结果默认条数 * @return {@link null} * @created by wWei * @date 2023/4/10 11:03 */ - public DruidSampleSQLHelper(String originalSQL, String filterExpr, boolean isInvalidCondExpr) { + public SampleSQLHelper(String originalSQL, String filterExpr, Boolean isInvalidCondExpr, Integer defaultSize) { this.originalSQL = originalSQL; this.filterExpr = filterExpr; this.isInvalidCondExpr = isInvalidCondExpr; + this.maxSize = defaultSize; } /** @@ -81,6 +86,7 @@ public class DruidSampleSQLHelper { } } + private class SelectAdapter extends SelectVisitorAdapter { @SneakyThrows @Override @@ -97,24 +103,43 @@ public class DruidSampleSQLHelper { fromItem.accept(new FromItemAdapter()); } - if ((fromItem instanceof Table) && plainSelect.getWhere() != null) { - plainSelect.setWhere(CCJSqlParserUtil.parseCondExpression(filterExpr)); + if ((fromItem instanceof Table) && filterExpr != null) { + Expression expression = CCJSqlParserUtil.parseCondExpression(filterExpr, false); + if (plainSelect.getWhere() != null) { + Expression where = plainSelect.getWhere(); + if (isInvalidCondExpr) { + OrExpression orExpression = buildOrExpression(where); + plainSelect.setWhere(orExpression); + where = plainSelect.getWhere(); + } + AndExpression andExpression = new AndExpression(); + Parenthesis parenthesisLeft = new Parenthesis(); + parenthesisLeft.setExpression(where instanceof Parenthesis ? ((Parenthesis) where).getExpression() : where); + Parenthesis parenthesisRight = new Parenthesis(); + parenthesisRight.setExpression(expression); + andExpression.setLeftExpression(parenthesisLeft); + andExpression.setRightExpression(parenthesisRight); + plainSelect.setWhere(andExpression); + } else { + plainSelect.setWhere(expression); + } } Expression having = plainSelect.getHaving(); if (having != null) { if (isInvalidCondExpr) { - OrExpression orExpression = new OrExpression(); - orExpression.setRightExpression(having); - EqualsTo equalsTo = new EqualsTo(); - LongValue longValue = new LongValue(1); - equalsTo.setLeftExpression(longValue); - equalsTo.setRightExpression(longValue); - orExpression.setLeftExpression(equalsTo); + OrExpression orExpression = buildOrExpression(having); plainSelect.setHaving(orExpression); } having.accept(new ExpressionAdapter()); } + + Limit limit = plainSelect.getLimit(); + if (limit == null && maxSize != null) { + Limit limitObj = new Limit(); + limitObj.setRowCount(new LongValue(maxSize)); + plainSelect.setLimit(limitObj); + } } @Override @@ -125,5 +150,16 @@ public class DruidSampleSQLHelper { } } } + + private OrExpression buildOrExpression(Expression where) { + OrExpression orExpression = new OrExpression(); + orExpression.setRightExpression(where); + EqualsTo equalsTo = new EqualsTo(); + LongValue longValue = new LongValue(1); + equalsTo.setLeftExpression(longValue); + equalsTo.setRightExpression(longValue); + orExpression.setLeftExpression(equalsTo); + return orExpression; + } } } diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/ClickhouseSampleSQLHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/SampleSQLHelperOfDataset.java index 34887778..14b6a719 100644 --- a/src/main/java/com/mesalab/common/utils/sqlparser/ClickhouseSampleSQLHelper.java +++ b/src/main/java/com/mesalab/common/utils/sqlparser/SampleSQLHelperOfDataset.java @@ -17,11 +17,12 @@ import java.util.List; /** * TODO * - * @Classname ClickhouseSampleSQLHelper + * @Classname SampleSQLHelperOfDataset * @Date 2023/3/22 10:42 * @Author wWei */ -public class ClickhouseSampleSQLHelper { +@Deprecated +public class SampleSQLHelperOfDataset { private final String templateSQL; private final String originalSQL; @@ -39,7 +40,7 @@ public class ClickhouseSampleSQLHelper { * @created by wWei * @date 2023/3/23 15:17 */ - public ClickhouseSampleSQLHelper(String originalSQL, String dataSetTemplate, boolean isInvalidCondExpr) { + public SampleSQLHelperOfDataset(String originalSQL, String dataSetTemplate, boolean isInvalidCondExpr) { this.templateSQL = dataSetTemplate; this.originalSQL = originalSQL; this.isInvalidCondExpr = isInvalidCondExpr; @@ -63,6 +64,9 @@ public class ClickhouseSampleSQLHelper { @Override public void visit(Table table) { + if (StrUtil.isEmptyIfStr(templateSQL)) { + return; + } if (table.getAlias() == null) { table.setAlias(new Alias(table.getName(), true)); } diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/SelectItemHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/SelectItemHelper.java new file mode 100644 index 00000000..20dcfce5 --- /dev/null +++ b/src/main/java/com/mesalab/common/utils/sqlparser/SelectItemHelper.java @@ -0,0 +1,143 @@ +package com.mesalab.common.utils.sqlparser; + +import com.google.common.collect.Maps; +import com.mesalab.qgw.constant.DataTypeConst; +import lombok.Data; +import lombok.SneakyThrows; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.*; + +import java.util.*; + +/** + * TODO + * + * @Classname SelectItemHelper + * @Date 2024/2/27 09:48 + * @Author wWei + */ +public class SelectItemHelper { + + private SelectItemHelper() { + } + + + public static Map<String, AliasObject> getSelectItem(String sql) throws JSQLParserException { + Select select = (Select) CCJSqlParserUtil.parse(sql); + SelectAdapter selectAdapter = new SelectAdapter(); + select.getSelectBody().accept(selectAdapter); + return selectAdapter.getAliasObjects(); + } + + private static class SelectAdapter extends SelectVisitorAdapter { + + private final Map<String, AliasObject> aliasObjects = Maps.newHashMap(); + + public Map<String, AliasObject> getAliasObjects() { + return aliasObjects; + } + + @SneakyThrows + @Override + public void visit(PlainSelect plainSelect) { + List<SelectItem> selectItems = plainSelect.getSelectItems(); + if (selectItems == null) { + return; + } + for (int i = 0; i < selectItems.size(); i++) { + SelectItem selectItem = selectItems.get(i); + SelectItemAdapter selectItemAdapter = new SelectItemAdapter(); + selectItem.accept(selectItemAdapter); + AliasObject aliasExpr = selectItemAdapter.getAliasExpr(); + if (aliasExpr != null) { + aliasExpr.setIndex(i); + this.aliasObjects.put(aliasExpr.getName(), aliasExpr); + } + } + } + + @Override + public void visit(SetOperationList setOpList) { + SelectBody selectBody = setOpList.getSelects().get(0); + if (selectBody instanceof PlainSelect) { + selectBody.accept(this); + } + } + } + + private static class SelectItemAdapter extends SelectItemVisitorAdapter { + + private AliasObject aliasExpr = new AliasObject(); + + public AliasObject getAliasExpr() { + return aliasExpr; + } + + @Override + public void visit(SelectExpressionItem selectExpressionItem) { + Expression expression = selectExpressionItem.getExpression(); + String name = selectExpressionItem.getAlias() != null ? selectExpressionItem.getAlias().getName() : selectExpressionItem.toString(); + name = SQLHelper.removeQuotesAndBackticks(name); + if (expression instanceof Column) { + Column column = (Column) expression; + AliasColumn aliasColumn = new AliasColumn(); + aliasColumn.setName(name); + aliasColumn.setFieldName(column.getColumnName()); + this.aliasExpr = aliasColumn; + return; + } else if (expression instanceof Function) { + Function function = (Function) expression; + String functionName = function.getName().toUpperCase(); + Map<String, String> functionDateTypeMap = Maps.newHashMap(); + functionDateTypeMap.put("FROM_UNIXTIME", DataTypeConst.TIMESTAMP_FORMAT); + functionDateTypeMap.put("DATE_FORMAT", DataTypeConst.TIMESTAMP_FORMAT); + functionDateTypeMap.put("CONVERT_TZ", DataTypeConst.TIMESTAMP_FORMAT); + functionDateTypeMap.put("TIME_FORMAT", DataTypeConst.TIMESTAMP_FORMAT); + functionDateTypeMap.put("FROM_UNIXTIME_MILLIS", DataTypeConst.TIMESTAMP_MS_FORMAT); + functionDateTypeMap.put("UNIX_TIMESTAMP", DataTypeConst.UNIX_TIMESTAMP); + functionDateTypeMap.put("TIME_FLOOR_WITH_FILL", DataTypeConst.UNIX_TIMESTAMP); + functionDateTypeMap.put("UNIX_TIMESTAMP_MILLIS", DataTypeConst.UNIX_TIMESTAMP_MS); + functionDateTypeMap.put("PERCENTILES_HDR", DataTypeConst.HDR_HISTOGRAM); + functionDateTypeMap.put("IP_TO_GEO", DataTypeConst.IP); + functionDateTypeMap.put("IP_TO_CITY", DataTypeConst.IP); + functionDateTypeMap.put("IP_TO_COUNTRY", DataTypeConst.IP); + functionDateTypeMap.put("IP_TO_ISP", DataTypeConst.IP); + functionDateTypeMap.put("IP_TO_ASN", DataTypeConst.IP); + functionDateTypeMap.put("IP_TO_ASN_DETAIL", DataTypeConst.IP); + functionDateTypeMap.put("IP_TO_ASN_ORG", DataTypeConst.IP); + + if (functionDateTypeMap.containsKey(functionName)) { + AliasFunExpr aliasFunExpr = new AliasFunExpr(); + aliasFunExpr.setName(name); + aliasFunExpr.setDateType(functionDateTypeMap.get(functionName)); + this.aliasExpr = aliasFunExpr; + return; + } + } + AliasObject aliasObject = new AliasObject(); + aliasObject.setName(name); + this.aliasExpr = aliasObject; + } + + } + + @Data + public static class AliasObject { + private String name; + private int index; + } + + @Data + public static class AliasColumn extends AliasObject { + private String fieldName; + } + + @Data + public static class AliasFunExpr extends AliasObject { + private String dateType; + } +} diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/TopSQLVisitor.java b/src/main/java/com/mesalab/common/utils/sqlparser/TopSQLVisitor.java index faade965..d82ccc18 100644 --- a/src/main/java/com/mesalab/common/utils/sqlparser/TopSQLVisitor.java +++ b/src/main/java/com/mesalab/common/utils/sqlparser/TopSQLVisitor.java @@ -31,7 +31,7 @@ import java.util.stream.Collectors; public class TopSQLVisitor implements OrderByVisitor, GroupByVisitor, SelectItemVisitor { private static final Log log = LogFactory.get(); private static final long TOP_INNER_LIMIT_RATIO = 10; - private static final long TOP_INNER_LIMIT_LOWER = 100000; + private static final long TOP_INNER_LIMIT_LOWER = 1000; private static final Pattern pLimitSetting = Pattern.compile("\\blimit(.*?)(\\))", Pattern.CASE_INSENSITIVE); private List<String> groupByList; private List<String> orderByList; @@ -148,7 +148,9 @@ public class TopSQLVisitor implements OrderByVisitor, GroupByVisitor, SelectItem * @return */ private boolean parseAndValidation(PlainSelect plainSelect) { - + if (plainSelect == null) { + return false; + } for (SelectItem item : plainSelect.getSelectItems()) { item.accept(this); } diff --git a/src/main/java/com/mesalab/knowledge/controller/KnowledgeController.java b/src/main/java/com/mesalab/knowledge/controller/KnowledgeController.java index 44d9d2e9..55d4c853 100644 --- a/src/main/java/com/mesalab/knowledge/controller/KnowledgeController.java +++ b/src/main/java/com/mesalab/knowledge/controller/KnowledgeController.java @@ -3,14 +3,13 @@ package com.mesalab.knowledge.controller; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; import com.mesalab.knowledge.enums.QueryTypeEnum; -import com.mesalab.knowledge.exception.KnowLedgeErrorCode; import com.mesalab.knowledge.exception.KnowLedgeErrorMessage; import com.mesalab.knowledge.service.KnowledgeService; import com.geedgenetworks.utils.StringUtil; +import com.mesalab.common.exception.CommonErrorCode; import org.apache.commons.lang3.EnumUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; @@ -44,14 +43,14 @@ public class KnowledgeController { log.debug("服务推荐接口, 参数: queryString is {},params is {}", param, object); if (!EnumUtils.isValidEnum(QueryTypeEnum.class, StringUtil.upperCase(param)) || StringUtil.isNotEmpty(request.getParameter(param))) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), - ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.NOT_SUPPORT)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), + CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.NOT_SUPPORT)); } return knowledgeService.query(object); } - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), - ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.NOT_SUPPORT)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), + CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.NOT_SUPPORT)); } } diff --git a/src/main/java/com/mesalab/knowledge/service/KnowledgeService.java b/src/main/java/com/mesalab/knowledge/service/KnowledgeService.java index 69442081..84b70850 100644 --- a/src/main/java/com/mesalab/knowledge/service/KnowledgeService.java +++ b/src/main/java/com/mesalab/knowledge/service/KnowledgeService.java @@ -5,19 +5,12 @@ import cn.hutool.core.date.DateUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.alibaba.fastjson2.JSON; -import com.fasterxml.jackson.databind.JsonNode; -import com.github.fge.jackson.JsonLoader; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.core.report.ProcessingMessage; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchema; -import com.github.fge.jsonschema.main.JsonSchemaFactory; import com.google.common.base.Splitter; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.utils.JsonSchemaValidator; import com.mesalab.knowledge.entity.*; import com.mesalab.knowledge.enums.DataSourceEnum; import com.mesalab.knowledge.enums.MatchEnum; @@ -27,25 +20,18 @@ import com.mesalab.knowledge.strategy.QueryProvider; import com.mesalab.qgw.constant.QGWMessageConst; import com.mesalab.qgw.exception.QGWBusinessException; import com.geedgenetworks.utils.StringUtil; +import com.mesalab.common.exception.CommonErrorCode; import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.Validate; -import org.apache.commons.lang3.time.DateUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.ObjectUtils; import java.io.IOException; -import java.text.ParseException; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; import static java.util.Objects.*; @@ -88,17 +74,17 @@ public class KnowledgeService { DSLObject.DSLQuery dslQuery = dSLObject.getQuery(); if (ObjectUtils.isEmpty(dslQuery)){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.QUERY_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.QUERY_IS_INVALID)); } Parameters parameters = dslQuery.getParameters(); if (ObjectUtils.isEmpty(dslQuery.getDataSource())){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.DATASOURCE_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.DATASOURCE_IS_INVALID)); } if (ObjectUtils.isEmpty(EnumUtils.getEnum(DataSourceEnum.class,StringUtil.upperCase(dslQuery.getDataSource())))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.DATASOURCE_IS_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.DATASOURCE_IS_ERROR)); } if (isNull(parameters)) { @@ -113,15 +99,15 @@ public class KnowledgeService { if (!ObjectUtils.isEmpty(matchs)) { matchs.forEach(m -> { if (!EnumUtils.isValidEnum(MatchEnum.class, StringUtil.upperCase(m.getType()))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.MATCH_TYPE_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.MATCH_TYPE_ERROR)); } //正则匹配方式 禁止 $xxx* 形式的value if (MatchEnum.REGEX.getType().equals(m.getType())) { m.getFieldValues().forEach(mv -> { if((String.valueOf(mv).startsWith("$") && String.valueOf(mv).endsWith("*"))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.MATCHING_SIGN)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.MATCHING_SIGN)); } }); } @@ -130,16 +116,16 @@ public class KnowledgeService { if (!ObjectUtils.isEmpty(ranges)) { ranges.forEach(r -> { if (!EnumUtils.isValidEnum(RangeEnum.class, StringUtil.upperCase(r.getType()))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.RANGE_TYPE_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.RANGE_TYPE_ERROR)); } }); } if (!ObjectUtils.isEmpty(sorts)) { sorts.forEach(s -> { if (!("asc".equalsIgnoreCase(s.getType()) || "desc".equalsIgnoreCase(s.getType()))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.SORT_TYPE_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.SORT_TYPE_ERROR)); } }); } @@ -147,64 +133,37 @@ public class KnowledgeService { try { List<String> times = Splitter.on("/").splitToList(intervals.get(0)); if (ObjectUtils.isEmpty(times) || times.size() != 2) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.INTERVALS_IS_INVALID)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.INTERVALS_IS_INVALID)); } for (String dateTimeStr : times) { DateUtil.parse(dateTimeStr); } } catch (DateException e) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.TIME_FORMAT_ERROR)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.TIME_FORMAT_ERROR)); } } if (nonNull(limit)) { List<String> limits = Splitter.on(",").trimResults().omitEmptyStrings().splitToList(limit); if (ObjectUtils.isEmpty(limits) || (limit.contains(",") && limits.size() != 2)){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.LIMIT_IS_ILLEGAL)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.LIMIT_IS_ILLEGAL)); } for (String l : limits) { if (!(l.length() <= maxrows.length() && Integer.valueOf(maxrows) >= Integer.valueOf(l) && Integer.valueOf(l) > 0)){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), String.format(KnowLedgeErrorMessage.LIMIT_IS_ILLEGAL,maxrows))); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), String.format(KnowLedgeErrorMessage.LIMIT_IS_ILLEGAL,maxrows))); } } } } private void validateJson(String jsonString) { - JsonSchemaFactory factory = JsonSchemaFactory.byDefault(); - //通过jsonschemaFactory获取jsonnode对象 - try { - - JsonNode schemaNode = JsonLoader.fromResource("/dsl-validation.json"); - //通过jsonstr字符串获取对应的jsonnode对象 - JsonNode dataNode = JsonLoader.fromString(jsonString); - JsonSchema jsonSchema = factory.getJsonSchema(schemaNode); - //使用json-schema-validator中的jsonschema对象的validate方法对数据进行校验 - //获取处理的报告信息 - ProcessingReport processingReport = jsonSchema.validate(dataNode); - //获取完整的报告信息 - Iterator<ProcessingMessage> iterator = processingReport.iterator(); - StringBuffer sb = new StringBuffer(); - while (iterator.hasNext()) { - ProcessingMessage next = iterator.next(); - JsonNode jsonNode = next.asJson(); - sb.append("pointer on "); - sb.append(jsonNode.get("instance").get("pointer")); - sb.append(", "); - sb.append(next.getMessage()); - sb.append(". "); - } - //判断校验是否成功，如果为true成功 - Validate.isTrue(processingReport.isSuccess(), sb.toString()); - } catch (ProcessingException e) { - log.error(e); - } catch (IOException e) { - log.error(e); - } + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + jsonSchemaValidator.addRule("dsl-validation.json", "dsl") + .validateDSL(jsonString); } } diff --git a/src/main/java/com/mesalab/knowledge/strategy/BaseQueryProvider.java b/src/main/java/com/mesalab/knowledge/strategy/BaseQueryProvider.java index 4276aa72..a86b1766 100644 --- a/src/main/java/com/mesalab/knowledge/strategy/BaseQueryProvider.java +++ b/src/main/java/com/mesalab/knowledge/strategy/BaseQueryProvider.java @@ -4,16 +4,15 @@ import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.alibaba.fastjson2.JSON; import com.google.common.collect.ImmutableMap; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; import com.mesalab.knowledge.common.config.ArangoConfig; import com.mesalab.knowledge.common.utils.Constant; import com.mesalab.knowledge.common.utils.JwtCache; -import com.mesalab.knowledge.exception.KnowLedgeErrorCode; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.HttpResponseResult; import com.mesalab.qgw.service.impl.HttpClientService; -import org.apache.http.Header; -import org.apache.http.message.BasicHeader; +import com.mesalab.qgw.service.impl.HttpClientServiceV2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpHeaders; import org.springframework.stereotype.Component; @@ -30,12 +29,16 @@ import java.util.Objects; @Component public class BaseQueryProvider { private static final Log log = LogFactory.get(); + private static final int UNAUTHORIZED_CODE = 401; @Autowired ArangoConfig arangoConfig; @Autowired HttpClientService httpClientService; + @Autowired + HttpClientServiceV2 httpClientServiceV2; + /** * @return java.lang.String * @Author zhq @@ -50,8 +53,8 @@ public class BaseQueryProvider { Map<String, String> params = new HashMap<>(); params.put("username", arangoConfig.getUsername()); params.put("password", arangoConfig.getPin()); - String res = httpClientService.httpPost(arangoConfig.getJwturl(), JSON.toJSONString(params)); - Map resMap = JSON.parseObject(res, Map.class); + HttpResponseResult httpResponseResult = httpClientServiceV2.post(arangoConfig.getJwturl(), JSON.toJSONString(params)); + Map resMap = JSON.parseObject(httpResponseResult.getResponseBody(), Map.class); if (Objects.isNull(resMap.get(Constant.JWT))) { throw new BusinessException("获取Arango jwt 失败"); } @@ -69,24 +72,18 @@ public class BaseQueryProvider { **/ public Object queryArango(String aql) { Map queryMap = ImmutableMap.of("query", aql); - Header header = new BasicHeader(HttpHeaders.AUTHORIZATION, getJwt()); - String res = ""; - try { - res = httpClientService.httpPost(arangoConfig.getQueryurl(), JSON.toJSONString(queryMap), header); - } catch (BusinessException e) { - if (e.getMessage() != null && e.getMessage().contains("not authorized to execute this request")) { - //刷新jwt - JwtCache.remove(Constant.ARANGO_CACHE_JWT); - log.warn("Arango jwt 失效后重置"); - res = httpClientService.httpPost(arangoConfig.getQueryurl(), JSON.toJSONString(queryMap), new BasicHeader(HttpHeaders.AUTHORIZATION, getJwt())); - } else { - log.error(e.getMessage()); - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),e.getMessage())); - } + Map<String, String> headers = new HashMap<>(); + headers.put(HttpHeaders.AUTHORIZATION, getJwt()); + HttpResponseResult responseResult = null; + responseResult = httpClientServiceV2.post(arangoConfig.getQueryurl(), JSON.toJSONString(queryMap), headers); + Map jsonMap = JSON.parseObject(responseResult.getResponseBody(), Map.class); + if (jsonMap != null && jsonMap.get("code") != null && Integer.parseInt(jsonMap.get("code").toString()) == UNAUTHORIZED_CODE) { + JwtCache.remove(Constant.ARANGO_CACHE_JWT); + log.warn("Arango jwt 失效后重置"); + headers.put(HttpHeaders.AUTHORIZATION, getJwt()); + responseResult = httpClientServiceV2.post(arangoConfig.getQueryurl(), JSON.toJSONString(queryMap), headers); + jsonMap = JSON.parseObject(responseResult.getResponseBody(), Map.class); } - Map jsonMap = JSON.parseObject(res, Map.class); - Object result = jsonMap.get("result"); - return result; + return jsonMap.get("result"); } } diff --git a/src/main/java/com/mesalab/knowledge/strategy/FqdnProviderImpl.java b/src/main/java/com/mesalab/knowledge/strategy/FqdnProviderImpl.java index 8e25f0d7..5a800116 100644 --- a/src/main/java/com/mesalab/knowledge/strategy/FqdnProviderImpl.java +++ b/src/main/java/com/mesalab/knowledge/strategy/FqdnProviderImpl.java @@ -10,8 +10,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; import com.mesalab.knowledge.entity.DSLObject; import com.mesalab.knowledge.entity.Match; @@ -19,19 +18,17 @@ import com.mesalab.knowledge.entity.Parameters; import com.mesalab.knowledge.entity.Range; import com.mesalab.knowledge.entity.arango.IpLearningPath; import com.mesalab.knowledge.enums.RangeEnum; -import com.mesalab.knowledge.exception.KnowLedgeErrorCode; import com.mesalab.knowledge.exception.KnowLedgeErrorMessage; import com.mesalab.qgw.constant.QGWMessageConst; import com.geedgenetworks.utils.DateUtils; import com.geedgenetworks.utils.StringUtil; +import com.mesalab.common.exception.CommonErrorCode; import org.apache.commons.lang3.EnumUtils; -import org.joda.time.format.ISODateTimeFormat; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.ObjectUtils; -import java.text.MessageFormat; import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -78,12 +75,12 @@ public class FqdnProviderImpl implements QueryProvider { //查询深度 if ("DEPTH".equals(r.getFieldKey())) { if (ObjectUtils.isEmpty(r.getFieldValues())){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.FIELD_VALUES_FORMAT_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.FIELD_VALUES_FORMAT_ERROR)); } if (!Pattern.matches(depthregex, String.valueOf(r.getFieldValues().get(0)))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.FIELD_VALUES_FORMAT_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), KnowLedgeErrorMessage.FIELD_VALUES_FORMAT_ERROR)); } } else if ("PROTOCOL".equals(r.getFieldKey())) { List<String> protocols = Lists.newArrayList(); @@ -122,8 +119,8 @@ public class FqdnProviderImpl implements QueryProvider { intervalsb.append(" and e.LAST_FOUND_TIME >= ").append(startTime).append(" and ").append("e.LAST_FOUND_TIME < ").append(endTime); } } catch (DateException e) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.TIME_FORMAT_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.TIME_FORMAT_ERROR)); } } diff --git a/src/main/java/com/mesalab/knowledge/strategy/SubscriberIdProviderImpl.java b/src/main/java/com/mesalab/knowledge/strategy/SubscriberIdProviderImpl.java deleted file mode 100644 index 0e0fb389..00000000 --- a/src/main/java/com/mesalab/knowledge/strategy/SubscriberIdProviderImpl.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.mesalab.knowledge.strategy; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.mesalab.cn.enums.EngineTypeEnum; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.knowledge.entity.*; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.common.enums.EntityQueryType; -import com.mesalab.services.service.RelationService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -/** - * @description: - * @author: zhq - * @create: 2020-07-17 - **/ -@Component("SUBSCRIBER_ID_VIEW") -public class SubscriberIdProviderImpl implements QueryProvider { - - private static final Log log = LogFactory.get(); - @Autowired - private RelationService relationService; - - @Override - public Object query(DSLObject.DSLQuery dslQeruy) { - BaseResult realRelation = null; - try { - Parameters parameters = dslQeruy.getParameters(); - - ComDSLObject.Query.QueryBean queryBean = new ComDSLObject.Query.QueryBean(); - queryBean.setMatch(getComMatch(parameters)); - queryBean.setRange(getComRange(parameters)); - queryBean.setSort(getComSort(parameters)); - queryBean.setIntervals(parameters.getIntervals()); - - ComDSLObject.Query query = new ComDSLObject.Query(); - query.setDataSource(dslQeruy.getDataSource()); - query.setDataEngine(EngineTypeEnum.ANALYSIS_ENGINE.getEngine()); - query.setLimit(parameters.getLimit()); - query.setParameters(queryBean); - - ComDSLObject comDSLObject = new ComDSLObject(); - comDSLObject.setQuery(query); - - realRelation = relationService.getRealRelation(EntityQueryType.SUBSCRIBERIDPOOL.getType(), comDSLObject); - } catch (Exception e) { - log.error(e); - } - return realRelation.getData(); - } - - private List<ComDSLObject.Query.FilterBean> getComRange(Parameters parameters) throws Exception { - List<ComDSLObject.Query.FilterBean> range = new ArrayList<>(); - if (parameters.getRange() != null) { - for (Range parametersRange : parameters.getRange()) { - ComDSLObject.Query.FilterBean filterBean = new ComDSLObject.Query.FilterBean(); - filterBean.setType(parametersRange.getType()); - filterBean.setFieldKey(parametersRange.getFieldKey()); - filterBean.setFieldValues(parametersRange.getFieldValues().stream() - .map(String::valueOf) - .collect(Collectors.toList())); - range.add(filterBean); - } - } - - return range; - } - - private List<ComDSLObject.Query.FilterBean> getComMatch(Parameters parameters) throws Exception { - List<ComDSLObject.Query.FilterBean> match = new ArrayList<>(); - if (parameters.getMatch() != null) { - for (Match parametersMatch : parameters.getMatch()) { - ComDSLObject.Query.FilterBean filterBean = new ComDSLObject.Query.FilterBean(); - filterBean.setType(parametersMatch.getType()); - filterBean.setFieldKey(parametersMatch.getFieldKey()); - filterBean.setFieldValues(parametersMatch.getFieldValues().stream() - .map(String::valueOf) - .collect(Collectors.toList())); - match.add(filterBean); - } - } - - return match; - } - - private List<ComDSLObject.Query.SortBean> getComSort(Parameters parameters) throws Exception { - List<ComDSLObject.Query.SortBean> sort = new ArrayList<>(); - if (parameters.getSort() != null) { - for (Sort parametersSort : parameters.getSort()) { - ComDSLObject.Query.SortBean sortBean = new ComDSLObject.Query.SortBean(); - sortBean.setType(parametersSort.getType()); - sortBean.setFieldKey(parametersSort.getFieldKey()); - sort.add(sortBean); - } - } - return sort; - } - -}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/network/controller/NetworkMonitorController.java b/src/main/java/com/mesalab/network/controller/NetworkMonitorController.java deleted file mode 100644 index 3e58d35d..00000000 --- a/src/main/java/com/mesalab/network/controller/NetworkMonitorController.java +++ /dev/null @@ -1,115 +0,0 @@ -package com.mesalab.network.controller; - -import cn.hutool.core.util.StrUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.google.common.collect.Lists; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.network.dsl.DSLObject; -import com.mesalab.network.dsl.DSLValidate; -import com.mesalab.network.exception.NWErrorCode; -import com.mesalab.network.service.NetworkMonitorService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - -import javax.servlet.http.HttpServletRequest; -import java.util.Enumeration; -import java.util.List; -import java.util.Locale; - - -/** - * @author wangwei - * @version 1.0 - * @date 2020/6/30 10:38 上午 - */ -@RestController -@RequestMapping(value = "/") -public class NetworkMonitorController { - private static final Log log = LogFactory.get(); - @Autowired - private DSLValidate dslValidate; - @Autowired - private NetworkMonitorService networkMonitorService; - private static final String PROTOCOL = "protocol"; - private static final String APP = "app"; - - @PostMapping(value = "/traffic/v1/", produces = "application/json") - public BaseResult trafficStatistics(HttpServletRequest request, @Validated @RequestBody DSLObject dslObject) { - Enumeration<String> parameterNames = request.getParameterNames(); - List<String> params = Lists.newArrayList(); - while (parameterNames.hasMoreElements()) { - params.add(StrUtil.trim(parameterNames.nextElement()).toLowerCase(Locale.ROOT)); - } - if (params.contains(PROTOCOL)) { - log.debug("network protocol interface, param: queryString is {}, body is {}.", params, dslObject); - dslValidate.executeValidate(dslObject); - return protocolData(dslObject); - } else if (params.contains(APP)) { - log.debug("network app interface, param: queryString is {}, body is {}.", params, dslObject); - dslValidate.executeValidate(dslObject); - return appData(dslObject); - } else { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), "not support query parameter"); - } - } - - private BaseResult appData(DSLObject dslObject) { - String queryType = dslObject.getQuery().getQueryType(); - if (QueryType.APP_DATA_SUMMARY.getValue() - .equalsIgnoreCase(queryType)) { - return networkMonitorService.buildAppData(dslObject); - } else if (QueryType.INTERNAL_IP_DATA_SUMMARY.getValue() - .equalsIgnoreCase(queryType)) { - return networkMonitorService.buildInternalIPData(dslObject); - } - else if (QueryType.APP_DATA_RATE_SUMMARY.getValue() - .equalsIgnoreCase(queryType)) { - return networkMonitorService.buildAppDataRate(dslObject); - } else if (QueryType.APP_TRAFFIC_SUMMARY.getValue() - .equalsIgnoreCase(queryType)) { - return networkMonitorService.buildAppTraffic(dslObject); - } else { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), "No match queryType"); - } - } - - private BaseResult protocolData(DSLObject dslObject) { - if (QueryType.PROTOCOL_TREE_SUMMARY.getValue() - .equalsIgnoreCase(dslObject.getQuery().getQueryType())) { - return networkMonitorService.buildProtocolTree(dslObject); - } else if (QueryType.PROTOCOL_DATA_RATE_SUMMARY.getValue() - .equalsIgnoreCase(dslObject.getQuery().getQueryType())) { - return networkMonitorService.buildProtocolDataRateSummary(dslObject); - } else if (QueryType.NETWORK_OVERVIEW_SUMMARY.getValue().equalsIgnoreCase(dslObject.getQuery().getQueryType())) { - return networkMonitorService.buildNetworkOverviewResult(dslObject); - } else { - return BaseResultGenerator.failure(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), "No match queryType"); - } - } - - enum QueryType { - PROTOCOL_TREE_SUMMARY("protocolTreeSummary"), - PROTOCOL_DATA_RATE_SUMMARY("protocolDataRateSummary"), - NETWORK_OVERVIEW_SUMMARY("networkOverviewSummary"), - APP_DATA_SUMMARY("appDataSummary"), - APP_DATA_RATE_SUMMARY("appDataRateSummary"), - APP_TRAFFIC_SUMMARY("appTrafficSummary"), - - INTERNAL_IP_DATA_SUMMARY("internalIPDataSummary"); - private final String value; - - QueryType(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - } -} diff --git a/src/main/java/com/mesalab/network/dsl/DSLObject.java b/src/main/java/com/mesalab/network/dsl/DSLObject.java deleted file mode 100644 index 59e45a65..00000000 --- a/src/main/java/com/mesalab/network/dsl/DSLObject.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.mesalab.network.dsl; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @Date: 2020-09-18 09:48 - * @Author : wangwei - * @ClassName : DSLObject - * @Description : - */ -@Data -public class DSLObject implements Serializable { - - private String clientId; - private QueryBean query; - - @Data - public static class QueryBean { - private String queryType; - private String dataSource; - private Integer limit; - private Integer offset; - private Parameters parameters; - - @Data - public static class Parameters { - private String granularity; - private List<FilterBean> match; - private List<FilterBean> range; - private List<String> intervals; - - @Data - public static class FilterBean { - private String type; - private String fieldKey; - private List<String> fieldValues; - } - } - } -} diff --git a/src/main/java/com/mesalab/network/dsl/DSLValidate.java b/src/main/java/com/mesalab/network/dsl/DSLValidate.java deleted file mode 100644 index 34bf3ed5..00000000 --- a/src/main/java/com/mesalab/network/dsl/DSLValidate.java +++ /dev/null @@ -1,162 +0,0 @@ -package com.mesalab.network.dsl; - -import cn.hutool.core.date.DateException; -import cn.hutool.core.date.DateUtil; -import cn.hutool.core.util.NumberUtil; -import com.mesalab.cn.constant.ErrorMessage; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.knowledge.enums.MatchEnum; -import com.mesalab.knowledge.enums.RangeEnum; -import com.mesalab.network.exception.NWErrorCode; -import com.mesalab.network.exception.NWErrorMessage; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.geedgenetworks.utils.StringUtil; -import org.apache.commons.lang.Validate; -import org.apache.commons.lang3.EnumUtils; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.List; -import java.util.regex.Pattern; - -/** - * @Date: 2020-09-17 18:10 - * @Author : wangwei - * @ClassName : DSLValidate - * @Description : DSL格式校验 - */ -@Component -public class DSLValidate { - - public static final Pattern periodOfPT = Pattern.compile("PT(\\d+)[SMH]", Pattern.CASE_INSENSITIVE); - public static final Pattern periodOfP = Pattern.compile("P(\\d+)[DWMY]", Pattern.CASE_INSENSITIVE); - - public void executeValidate(DSLObject dslObject) throws BusinessException { - if (StringUtil.isEmpty(dslObject)) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),NWErrorMessage.DSL_OBJECT_IS_INVALID)); - } - if (StringUtil.isEmpty(dslObject.getQuery())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),NWErrorMessage.QUERY_IS_INVALID)); - } - if (StringUtil.isNotEmpty(dslObject.getQuery().getLimit()) && !NumberUtil.isNumber(dslObject.getQuery().getLimit().toString())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),NWErrorMessage.QUERY_LIMIT_IS_INVALID)); - } - if (StringUtil.isNotEmpty(dslObject.getQuery().getOffset()) && !NumberUtil.isNumber(dslObject.getQuery().getOffset().toString())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),NWErrorMessage.QUERY_OFFSET_IS_INVALID)); - } - DSLObject.QueryBean.Parameters parameters = dslObject.getQuery().getParameters(); - if (StringUtil.isEmpty(parameters)) { - return; - } - if (!isValidGranularity(parameters.getGranularity())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),NWErrorMessage.GRANULARITY_VALUE_IS_INVALID)); - } - if (!isValidMatch(parameters.getMatch())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),NWErrorMessage.MATCH_IS_INVALID)); - } - if (!isValidRange(parameters.getRange())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),NWErrorMessage.RANGE_TYPE_IS_INVALID)); - } - if (!isValidIntervals(parameters.getIntervals())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),NWErrorMessage.INTERVALS_TYPE_IS_INVALID)); - } - } - - /** - * 时间粒度校验, 遵循ISO8601 durations 定义 - * - * @param granularity - * @return - */ - private boolean isValidGranularity(String granularity) { - if (StringUtil.isBlank(granularity)) { - return true; - } - if (periodOfP.matcher(granularity).find() || periodOfPT.matcher(granularity).find()) { - return true; - } - return false; - } - - /** - * 校验match: - * 1.是否属于{@link MatchEnum}限定类型 - * 2.不能以*开始、或$结尾 - * - * @param filterBeanList - * @return - */ - private boolean isValidMatch(List<DSLObject.QueryBean.Parameters.FilterBean> filterBeanList) { - if (CollectionUtils.isEmpty(filterBeanList)) { - return true; - } - for (DSLObject.QueryBean.Parameters.FilterBean match : filterBeanList) { - Validate.isTrue(EnumUtils.isValidEnum(MatchEnum.class, StringUtil.upperCase(match.getType())), ErrorMessage.MATCH_FIELD_VALUES_ERROR); - } - return true; - } - - /** - * 校验range: - * 1.是否属于{@link RangeEnum}限定类型 - * - * @param filterBeanList - * @return - */ - private boolean isValidRange(List<DSLObject.QueryBean.Parameters.FilterBean> filterBeanList) { - if (CollectionUtils.isEmpty(filterBeanList)) { - return true; - } - for (DSLObject.QueryBean.Parameters.FilterBean range : filterBeanList) { - if (!EnumUtils.isValidEnum(RangeEnum.class, StringUtil.upperCase(range.getType()))){ - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),NWErrorMessage.RANGE_TYPE_ERROR)); - } - } - return true; - } - - /** - * 校验Interval: - * 1.目前只支持between类型: ["2020-01-01 00:00:00/2020-01-02 00:00:00"] - * 2.时间区间必须是 开始时间 < 结束时间 - * - * @param intervals - */ - private boolean isValidIntervals(List<String> intervals) { - try { - if (CollectionUtils.isEmpty(intervals)) { - return true; - } - if (intervals.size() != 1) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), NWErrorMessage.INTERVALS_PARAM_ERROR)); - } - String[] split = intervals.get(0).split("/"); - if (split.length != 2) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), NWErrorMessage.INTERVALS_PARAM_ERROR)); - } - for (String dateTimeStr : split) { - DateUtil.parse(dateTimeStr); - } - return true; - } catch (DateException e) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), NWErrorMessage.TIME_FORMAT_ERROR)); - } - } - -} diff --git a/src/main/java/com/mesalab/network/exception/NWErrorCode.java b/src/main/java/com/mesalab/network/exception/NWErrorCode.java deleted file mode 100644 index 1774b1dd..00000000 --- a/src/main/java/com/mesalab/network/exception/NWErrorCode.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.mesalab.network.exception; - -import lombok.Getter; - -/** - * @Author wxs - * @Date 2022/11/3 - */ -@Getter -public enum NWErrorCode { - /** - * 1-3位：异常类型（HTTP协议状态码） - * 4-5位：模块 04 - * 6-8：自然排序 - */ - - - - - ; - - - private String code; - private String message; - - NWErrorCode(String code, String message) { - this.code = code; - this.message = message; - } - -}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/network/exception/NWErrorMessage.java b/src/main/java/com/mesalab/network/exception/NWErrorMessage.java deleted file mode 100644 index 6176fb6e..00000000 --- a/src/main/java/com/mesalab/network/exception/NWErrorMessage.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.mesalab.network.exception; - - -/** - * @Author wxs - * @Date 2022/11/3 - */ -public class NWErrorMessage { - /** - * 1-2位：异常类型（参数检查异常:01、SQL语句检查异常:02、SQL执行异常:03、引擎计算异常:04、解析异常:05、执行失败:99）。 - * 3-4位：模块 03 - * 5-7：自然排序 - */ - public static String DSL_OBJECT_IS_INVALID = "DSLObject is invalid"; - public static String QUERY_IS_INVALID = "DSLObject.query is invalid"; - public static String QUERY_LIMIT_IS_INVALID = "DSLObject.query.limit is invalid"; - public static String QUERY_OFFSET_IS_INVALID = "DSLObject.query.offset is invalid"; - public static String GRANULARITY_VALUE_IS_INVALID = "DSLObject.query.parameters.Granularity value is invalid"; - public static String MATCH_IS_INVALID = "DSLObject.query.Match is invalid"; - public static String RANGE_TYPE_IS_INVALID = "DSLObject.query.Range type is invalid"; - public static String INTERVALS_TYPE_IS_INVALID = "DSLObject.query.Intervals type is invalid"; - public static String MATCH_FIELD_VALUES_ERROR = "Match fieldValues cannot startWith '*' or endWith '$'"; - public static String RANGE_TYPE_ERROR = "range type is illegal"; - public static String INTERVALS_PARAM_ERROR = "DSLObject.query.query.Intervals is invalid"; - public static String TIME_FORMAT_ERROR = "Time format should follow ISO 8601 standard."; - public static String INTERVALS_VALUE_ERROR = "Intervals value should be [start, end]"; - - -} diff --git a/src/main/java/com/mesalab/network/service/NetworkMonitorService.java b/src/main/java/com/mesalab/network/service/NetworkMonitorService.java deleted file mode 100644 index b054aa3e..00000000 --- a/src/main/java/com/mesalab/network/service/NetworkMonitorService.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.mesalab.network.service; - -import com.mesalab.common.entity.BaseResult; -import com.mesalab.network.dsl.DSLObject; -import com.mesalab.network.model.protocol.ProtocolTree; - -import java.util.List; -import java.util.Map; - -/** - * @author wangwei - * @version 1.0 - * @date 2020/6/30 4:20 下午 - */ - - -public interface NetworkMonitorService { - - BaseResult buildProtocolTree(DSLObject dslObject); - BaseResult buildProtocolDataRateSummary(DSLObject dslObject); - BaseResult buildNetworkOverviewResult(DSLObject dslObject); - List<ProtocolTree> buildFlatStructure(List<Map> data); - List<ProtocolTree> buildHierarchicalStructure(List<ProtocolTree> nodes); - BaseResult buildAppData(DSLObject dslObject); - BaseResult buildInternalIPData(DSLObject dslObject); - - BaseResult buildAppDataRate(DSLObject dslObject); - - BaseResult buildAppTraffic(DSLObject dslObject); -} diff --git a/src/main/java/com/mesalab/network/service/impl/NetworkMonitorServiceImpl.java b/src/main/java/com/mesalab/network/service/impl/NetworkMonitorServiceImpl.java deleted file mode 100644 index 6315f44e..00000000 --- a/src/main/java/com/mesalab/network/service/impl/NetworkMonitorServiceImpl.java +++ /dev/null @@ -1,607 +0,0 @@ -package com.mesalab.network.service.impl; - -import cn.hutool.core.net.NetUtil; -import cn.hutool.core.util.NumberUtil; -import cn.hutool.core.util.ObjectUtil; -import cn.hutool.core.util.StrUtil; -import cn.hutool.core.util.URLUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONWriter; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.mesalab.cn.enums.RangeTypeEnum; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.utils.TreeUtils; -import com.mesalab.qgw.model.basic.HttpConfig; -import com.mesalab.knowledge.enums.MatchEnum; -import com.mesalab.network.common.Constants; -import com.mesalab.network.dsl.DSLObject; -import com.mesalab.network.service.NetworkMonitorService; -import com.mesalab.network.model.protocol.ProtocolTree; -import com.mesalab.qgw.service.impl.HttpClientService; -import com.mesalab.services.common.dsl.ComDSLParse; -import com.mesalab.services.common.property.SqlPropertySourceFactory; -import com.geedgenetworks.utils.StringUtil; -import org.joda.time.Period; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.PropertySource; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - -import java.io.*; -import java.math.BigDecimal; -import java.net.URLEncoder; -import java.util.*; -import java.util.stream.Collectors; - -/** - * @author wangwei - * @version 1.0 - * @date 2020/6/30 4:24 下午 - */ -@Service("networkMonitorService") -@PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) -public class NetworkMonitorServiceImpl implements NetworkMonitorService { - private static final Log log = LogFactory.get(); - @Autowired - HttpClientService httpClientService; - @Autowired - HttpConfig httpConfig; - @Autowired - Environment environment; - private static final String localHostAddress; - - public String getPort() { - return environment.getProperty("local.server.port"); - } - private static final Map<String, String> LOGICAL_FIELD = Maps.newHashMap(); - - static { - localHostAddress = NetUtil.getLocalhostStr(); - LOGICAL_FIELD.put("protocol_stack_id", "protocol_path"); - LOGICAL_FIELD.put("app_name", "app"); - } - - @Override - public BaseResult buildProtocolTree(DSLObject dslObject) { - BaseResult baseResult; - String sql = generateProtocolTreeSql(dslObject.getQuery(), environment.getProperty("PROTOCOL_TREE_SUMMARY")); - Map<String, String> result = executeQuery(sql); - if (String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(result.get("status"))) { - Map resultMap = JSON.parseObject(result.get("result"), Map.class); - List<Map> resultList = (List<Map>) resultMap.get("data"); - if (StringUtil.isEmpty(resultList)) { - return BaseResultGenerator.success("ok", resultList, (Map) resultMap.get("statistics")); - } - List<ProtocolTree> protocolTreeList = buildHierarchicalStructure(buildFlatStructure(resultList)); - baseResult = BaseResultGenerator.success("ok", protocolTreeList, (Map) resultMap.get("statistics")); - } else { - baseResult = BaseResultGenerator.error(result.get("result")); - } - return baseResult; - } - - @Override - public BaseResult buildNetworkOverviewResult(DSLObject dslObject) { - Map<String, String> asymmetricFlowsRsp = executeQuery(generateNetworkStatSummarySql(dslObject.getQuery(), environment.getProperty("NETWORK_OVERVIEW_ASYMMETRIC_FLOWS_STAT"))); - if (!String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(asymmetricFlowsRsp.get("status"))) { - throw new BusinessException(Integer.parseInt(asymmetricFlowsRsp.get("status")), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), asymmetricFlowsRsp.get("message"))); - } - Map<String, String> statRsp = executeQuery(generateDynamicStatSql(dslObject.getQuery(), environment.getProperty("NETWORK_OVERVIEW_STAT"))); - if (!String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(statRsp.get("status"))) { - throw new BusinessException(Integer.parseInt(statRsp.get("status")), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), asymmetricFlowsRsp.get("message"))); - } - Map<String, String> tcpRsp = executeQuery(generateTCPStatSql(dslObject.getQuery(), environment.getProperty("NETWORK_OVERVIEW_TCP_STAT"))); - if (!String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(statRsp.get("status"))) { - throw new BusinessException(Integer.parseInt(statRsp.get("status")), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), asymmetricFlowsRsp.get("message"))); - } - Map<String, String> appRsp = executeQuery(generateAppStatisticsSql(dslObject.getQuery(), environment.getProperty("NETWORK_OVERVIEW_APP_STAT"))); - if (!String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(appRsp.get("status"))) { - throw new BusinessException(Integer.parseInt(appRsp.get("status")), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), asymmetricFlowsRsp.get("message"))); - } - - Map statResult = JSON.parseObject(statRsp.get("result"), Map.class); - List<Map> statData = (List<Map>) statResult.get("data"); - Map asymmetricFlowsResult = JSON.parseObject(asymmetricFlowsRsp.get("result"), Map.class); - List<Map> asymmetricFlowsData = (List<Map>) asymmetricFlowsResult.get("data"); - Map tcpResult = JSON.parseObject(tcpRsp.get("result"), Map.class); - List<Map> tcpData = (List<Map>) tcpResult.get("data"); - Map appResult = JSON.parseObject(appRsp.get("result"), Map.class); - List<Map> appData = (List<Map>) appResult.get("data"); - - Map<String, Object> data = new HashMap<>(16); - if (!statData.isEmpty()) { - data.putAll(statData.get(0)); - } - - if (!asymmetricFlowsData.isEmpty()) { - data.putAll(asymmetricFlowsData.get(0)); - } - - if (!tcpData.isEmpty()) { - data.putAll(tcpData.get(0)); - } - - if (!appData.isEmpty()) { - data.putAll(appData.get(0)); - } - - - long asymmetricFlows = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_BYTES) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_BYTES))); - long totalSessionUseOnAsymmetricFlows = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES_USE_ON_ASYMMETRIC_FLOWS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES_USE_ON_ASYMMETRIC_FLOWS))); - - long uniqClientIp = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_UNIQ_CLIENT_IP) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_UNIQ_CLIENT_IP))); - long totalSessions = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS))); - long totalBytes = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES))); - long totalPackets = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS))); - - long fragmentationPackets = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS))); - long dataRate = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_DATA_RATE) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_DATA_RATE))); - - long tcpRetransmissionsPackets = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PACKETS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PACKETS))); - long tcpTotalPackets = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TCP_TOTAL_PACKETS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TCP_TOTAL_PACKETS))); - long unknownAppBytes = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_BYTES) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_BYTES))); - - - - String unknownAppPercent = totalBytes == 0 ? String.valueOf(0) - : NumberUtil.div(BigDecimal.valueOf(unknownAppBytes), BigDecimal.valueOf(totalBytes), 4).toPlainString(); - String asymmetricFlowsPercent = totalSessionUseOnAsymmetricFlows == 0 ? String.valueOf(0) - : NumberUtil.div(BigDecimal.valueOf(asymmetricFlows), BigDecimal.valueOf(totalSessionUseOnAsymmetricFlows), 4).toPlainString(); - String tcpRetransmissionsPercent = tcpTotalPackets == 0 ? String.valueOf(0) - : NumberUtil.div(BigDecimal.valueOf(tcpRetransmissionsPackets), BigDecimal.valueOf(tcpTotalPackets), 4).toPlainString(); - String fragmentationPacketsPercent = totalPackets == 0 ? String.valueOf(0) - : NumberUtil.div(BigDecimal.valueOf(fragmentationPackets), BigDecimal.valueOf(totalPackets), 4).toPlainString(); - - List result = new ArrayList<>(); - Map resultMap = new LinkedHashMap<>(); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_UNIQ_CLIENT_IP, uniqClientIp); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS, totalSessions); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_DATA_RATE, dataRate); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES, totalBytes); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS, totalPackets); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_BYTES, unknownAppBytes); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_PERCENT, unknownAppPercent); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_BYTES, asymmetricFlows); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_PERCENT, asymmetricFlowsPercent); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PACKETS, tcpRetransmissionsPackets); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PERCENT, tcpRetransmissionsPercent); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS, fragmentationPackets); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PERCENT, fragmentationPacketsPercent); - result.add(resultMap); - - Map statistics = (Map) asymmetricFlowsResult.get("statistics"); - Map statStatistics = (Map) statResult.get("statistics"); - Map appStatistics = (Map) appResult.get("statistics"); - for (Object key : statistics.keySet()) { - if ("result_rows".equals(key.toString())) { - continue; - } - statistics.put(key, Long.parseLong(statistics.get(key).toString()) + Long.parseLong(statStatistics.get(key).toString()) + Long.parseLong(appStatistics.get(key).toString())); - } - - return BaseResultGenerator.success("ok", result, statistics); - } - - @Override - public BaseResult buildProtocolDataRateSummary(DSLObject dslObject) { - BaseResult baseResult; - String sql = generateProtocolDataRateSql(dslObject.getQuery(), environment.getProperty("PROTOCOL_DATA_RATE_SUMMARY")); - Map<String, String> result = executeQuery(sql); - if (String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(result.get("status"))) { - Map resultMap = JSON.parseObject(result.get("result"), Map.class); - List<Map> data = (List<Map>) resultMap.get("data"); - data.forEach(o -> { - String[] protocolIds = String.valueOf(o.get("type")).split("\\."); - String protocolId = protocolIds[protocolIds.length - 1]; - o.put("type", protocolId); - }); - baseResult = BaseResultGenerator.success("ok", data, (Map) resultMap.get("statistics")); - } else { - baseResult = BaseResultGenerator.error(result.get("result")); - } - return baseResult; - } - - private String generateDynamicStatSql(DSLObject.QueryBean queryParam, String sqlTemplate) { - String whereOfTime = getWhereOfTime(queryParam.getParameters()); - String whereOfExactly = getWhereFilter(queryParam.getParameters(), false, null); - String[] intervals = getIntervals(queryParam.getParameters().getIntervals()); - return String.format(sqlTemplate, - "UNIX_TIMESTAMP('".concat(intervals[1]).concat("')"), - "UNIX_TIMESTAMP('".concat(intervals[0]).concat("')"), - queryParam.getDataSource(), whereOfTime, whereOfExactly, Constants.PROTOCOL_ETHERNET_NODE); - } - - private String generateTCPStatSql(DSLObject.QueryBean queryParam, String sqlTemplate) { - String whereOfTime = getWhereOfTime(queryParam.getParameters()); - String whereOfExactly = getWhereFilter(queryParam.getParameters(), false, null); - return String.format(sqlTemplate, queryParam.getDataSource(), whereOfTime, whereOfExactly); - } - - private String generateAppStatisticsSql(DSLObject.QueryBean queryParam, String sqlTemplate) { - String whereOfTime = getWhereOfTime(queryParam.getParameters()); - String whereOfExactly = getWhereFilter(queryParam.getParameters(), false, null); - return String.format(sqlTemplate, queryParam.getDataSource(), whereOfTime, whereOfExactly); - } - - private String generateProtocolDataRateSql(DSLObject.QueryBean queryParam, String sqlTemplate) { - String whereOfTime = getWhereOfTime(queryParam.getParameters()); - String whereOfExactly = getWhereFilter(queryParam.getParameters(), false, Lists.newArrayList("protocol_stack_id")); - String protocolStr = null; - List<DSLObject.QueryBean.Parameters.FilterBean> match = queryParam.getParameters().getMatch(); - for (DSLObject.QueryBean.Parameters.FilterBean item : match) { - if (MatchEnum.PREFIX.getType().equals(item.getType())) { - String[] split = item.getFieldValues().get(0).split(","); - protocolStr = split[0].replaceFirst(Constants.PROTOCOL_TREE_ROOT_NAME + Constants.ENCAPSULATION_PATH_SEPARATOR, ""); - } - } - return String.format(sqlTemplate, - queryParam.getParameters().getGranularity(), queryParam.getDataSource(), whereOfTime, whereOfExactly, protocolStr, queryParam.getParameters().getGranularity(), - queryParam.getParameters().getGranularity(), queryParam.getDataSource(), whereOfTime, whereOfExactly, protocolStr, "%", - protocolStr.length() - protocolStr.replaceAll("\\.", "").length(), queryParam.getParameters().getGranularity()); - - } - - private String generateNetworkStatSummarySql(DSLObject.QueryBean queryParam, String sqlTemplate) { - String whereOfTime = getWhereOfTime(queryParam.getParameters()); - String whereOfExactly = getWhereFilter(queryParam.getParameters(), false, null); - return String.format(sqlTemplate, - whereOfTime, whereOfExactly); - } - - private String getWhereOfTime(DSLObject.QueryBean.Parameters parameters) { - if (CollectionUtils.isEmpty(parameters.getIntervals())) { - return StringUtil.EMPTY; - } - StringBuffer whereOfTime = new StringBuffer(); - String[] intervals = getIntervals(parameters.getIntervals()); - - whereOfTime.append("__time >= DATE_FORMAT(FROM_UNIXTIME(UNIX_TIMESTAMP('").append(intervals[0]).append("')), '%Y-%m-%d %H:%i:%s')") - .append("AND __time < DATE_FORMAT(FROM_UNIXTIME(UNIX_TIMESTAMP( '").append(intervals[1]).append("')), '%Y-%m-%d %H:%i:%s')"); - return whereOfTime.toString(); - } - - private String[] getIntervals(List<String> intervals) { - return intervals.get(0).split("/"); - } - - private String getWhereFilter(DSLObject.QueryBean.Parameters parameters, boolean isLogicalField, List<String> skipFields) { - List<String> filter = Lists.newArrayList(); - if (!CollectionUtils.isEmpty(parameters.getMatch())) { - filter.addAll(parseMatch(parameters.getMatch(), isLogicalField, skipFields)); - } - if (!CollectionUtils.isEmpty(parameters.getRange())) { - filter.addAll(parseRange(parameters.getRange(), isLogicalField, skipFields)); - } - String join = String.join(" ) AND ( ", filter); - return StringUtil.isEmpty(join) ? StringUtil.EMPTY : " AND " + " ( " + join + " ) "; - } - - public List<String> parseRange(List<DSLObject.QueryBean.Parameters.FilterBean> ranges, boolean isLogicalField, List<String> skipFields) { - if (ObjectUtil.isEmpty(ranges)) { - return Lists.newArrayList(); - } - List<String> list = Lists.newArrayList(); - for (DSLObject.QueryBean.Parameters.FilterBean range : ranges) { - if (StringUtil.isNotEmpty(skipFields) && skipFields.contains(range.getFieldKey())) { - continue; - } - List<String> values = Lists.newArrayList(); - String fieldKey = range.getFieldKey(); - fieldKey = isLogicalField && LOGICAL_FIELD.containsKey(fieldKey) ? LOGICAL_FIELD.get(fieldKey) : fieldKey; - String type = range.getType(); - List<String> fieldValues = range.getFieldValues(); - for (String value : fieldValues) { - if (!NumberUtil.isNumber(value)) { - value = "'".concat(ComDSLParse.encode(value)).concat("'"); - } - values.add(ComDSLParse.parseRange(value, type)); - } - String join = String.join((RangeTypeEnum.NE.getType().equalsIgnoreCase(type) ? " AND " : " OR ") + fieldKey, values); - if (StringUtil.isNotBlank(join)) { - list.add(fieldKey + join); - } - } - return list; - } - - public List<String> parseMatch(List<DSLObject.QueryBean.Parameters.FilterBean> matches, boolean isLogicalField, List<String> skipFields) { - if (ObjectUtil.isEmpty(matches)) { - return Lists.newArrayList(); - } - List<String> list = Lists.newArrayList(); - for (DSLObject.QueryBean.Parameters.FilterBean match : matches) { - if (StringUtil.isNotEmpty(skipFields) && skipFields.contains(match.getFieldKey())) { - continue; - } - List<String> values = Lists.newArrayList(); - String fieldKey = match.getFieldKey(); - fieldKey = isLogicalField && LOGICAL_FIELD.containsKey(fieldKey) ? LOGICAL_FIELD.get(fieldKey) : fieldKey; - String type = match.getType(); - List<String> fieldValues = match.getFieldValues(); - for (String value : fieldValues) { - values.add(ComDSLParse.parseMatch(ComDSLParse.encode(value), type)); - } - String join = String.join(" OR " + fieldKey, values); - if (StringUtil.isNotBlank(join)) { - list.add(fieldKey + join); - } - } - return list; - } - - private String generateProtocolTreeSql(DSLObject.QueryBean queryParam, String sqlTemplate) { - String whereOfTime = getWhereOfTime(queryParam.getParameters()); - String whereOfExactly = getWhereFilter(queryParam.getParameters(), false, null); - return String.format(sqlTemplate, - queryParam.getDataSource(), whereOfTime, whereOfExactly); - } - - @Override - public List<ProtocolTree> buildFlatStructure(List<Map> protocolData) { - List<ProtocolTree> nodes = Lists.newArrayList(); - for (Map protocolMap : protocolData) { - ProtocolTree protocolTree = convertStringToObject(JSON.toJSONString(protocolMap, JSONWriter.Feature.LargeObject)); - if (StringUtil.isNotEmpty(protocolTree)) { - nodes.add(protocolTree); - } - - } - ProtocolTree root = new ProtocolTree(Constants.PROTOCOL_TREE_ROOT_NAME, Constants.PROTOCOL_TREE_ROOT_NAME, null); - Map<String, Long> protocolsAndApplications = nodes.stream().collect(Collectors.groupingBy(ProtocolTree::getName, Collectors.counting())); - root.addMetric(Constants.PROTOCOL_TREE_METRIC_ENCAPSULATION_PATHS, nodes.size()); - root.addMetric(Constants.PROTOCOL_TREE_METRIC_PROTOCOLS_AND_APPLICATIONS, protocolsAndApplications.size()); - List<ProtocolTree> roots = nodes.stream().filter(o -> StringUtil.isBlank(o.getParentId())).collect(Collectors.toList()); - roots.forEach(item -> { - root.setSentBytes(root.getSentBytes() + item.getSentBytes()); - root.setReceivedBytes(root.getReceivedBytes() + item.getReceivedBytes()); - root.setTotalSentBytes(root.getTotalSentBytes() + item.getSentBytes()); - root.setTotalReceivedBytes(root.getTotalReceivedBytes() + item.getReceivedBytes()); - }); - nodes.forEach(item -> { - item.setId(Constants.PROTOCOL_TREE_ROOT_NAME + Constants.ENCAPSULATION_PATH_SEPARATOR + item.getId()); - }); - nodes.add(root); - return nodes; - } - - private Map<String, String> executeQuery(String sql) { - try { - sql = URLEncoder.encode(sql, "utf-8").replaceAll("\\+", "%20"); - } catch (UnsupportedEncodingException e) { - log.error("sql Encode error: ", e); - } - String queryURL = URLUtil.normalize(localHostAddress + ":" + getPort() + "/sql?query="); - int socketTimeOut = httpConfig.getServerResponseTimeOut(); - return httpClientService.httpGet(queryURL + sql, socketTimeOut); - } - - private ProtocolTree convertStringToObject(String protocolString) { - ProtocolTree protocolTree = null; - try { - Map<String, Object> results = JSON.parseObject(protocolString, Map.class); - String protocolId = String.valueOf(results.get("protocol_stack_id")); - if (StringUtil.isBlank(protocolId)) { - log.warn("Protocol Stack ID is NULL {}", protocolString); - return null; - } - long sessions = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_SESSIONS)).orElse(0).toString()); - long sentBytes = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_C2S_BYTES)).orElse(0).toString()); - long receivedBytes = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_S2C_BYTES)).orElse(0).toString()); - long sentPackets = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_C2S_PKTS)).orElse(0).toString()); - long receivedPackets = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_S2C_PKTS)).orElse(0).toString()); - List<String> protocols = Splitter.on(Constants.ENCAPSULATION_PATH_SEPARATOR).omitEmptyStrings().splitToList(protocolId); - String protocolName = StringUtil.isNotEmpty(protocols) ? protocols.get(protocols.size() - 1) : null; - protocolTree = new ProtocolTree(protocolId, protocolName, null); - protocolTree.setSentBytes(sentBytes); - protocolTree.setReceivedBytes(receivedBytes); - protocolTree.getMetrics().put(Constants.PROTOCOL_TREE_METRIC_SENT_PACKETS, sentPackets); - protocolTree.getMetrics().put(Constants.PROTOCOL_TREE_METRIC_RECEIVED_PACKETS, receivedPackets); - protocolTree.getMetrics().put(Constants.PROTOCOL_TREE_METRIC_SESSIONS, sessions); - } catch (BusinessException e) { - log.error("Convert Json String to Protocol Tree Object Error ", e); - } - - return protocolTree; - } - - @Override - public List<ProtocolTree> buildHierarchicalStructure(List<ProtocolTree> nodes) { - - List<ProtocolTree> targetNodes = groupByName(nodes); - - targetNodes.sort((o1, o2) -> { - long numThis = o1.getReceivedBytes() + o1.getSentBytes(); - long numParam = o2.getReceivedBytes() + o2.getSentBytes(); - if (numThis > numParam) { - return -1; - } else if (numThis < numParam) { - return 1; - } else { - return 0; - } - }); - Map<String, List<ProtocolTree>> protocolMap = Maps.newLinkedHashMap(); - List<ProtocolTree> roots = targetNodes.stream().filter(o -> StringUtil.isBlank(o.getParentId())).collect(Collectors.toList()); - protocolMap.put(Constants.PROTOCOLS_PARENT_ID, roots); - targetNodes.stream().filter(o -> StringUtil.isNotBlank(o.getParentId())).forEach(x -> { - if (StringUtil.isEmpty(protocolMap.get(x.getParentId()))) { - protocolMap.put(x.getParentId(), Lists.newArrayList(x)); - } else { - protocolMap.get(x.getParentId()).add(x); - } - }); - - List<ProtocolTree> protocolTreeList = TreeUtils.mergeTree(protocolMap, Constants.PROTOCOLS_PARENT_ID, ProtocolTree::getId, ProtocolTree::setChildrens); - for (ProtocolTree protocolTree : protocolTreeList) { - List<ProtocolTree> ethernetList = protocolTree.getChildrens().stream().filter( - p -> Constants.PROTOCOL_ETHERNET_NODE.equals(p.getName())).collect(Collectors.toList()); - if (protocolTree.getChildrens().size() != ethernetList.size()) { - protocolTree.setChildrens(ethernetList); - protocolTree.setReceivedBytes(StringUtil.isEmpty(ethernetList.get(0)) ? 0 : ethernetList.get(0).getReceivedBytes()); - protocolTree.setSentBytes(StringUtil.isEmpty(ethernetList.get(0)) ? 0 : ethernetList.get(0).getSentBytes()); - log.warn("Protocol Tree Exceptional data occurs and non-{} protocols are filtered out: {}", Constants.PROTOCOL_ETHERNET_NODE, protocolTree.getChildrens()); - } - } - - return protocolTreeList; - - } - - @Override - public BaseResult buildAppData(DSLObject dslObject) { - String whereOfExactly = getWhereFilter(dslObject.getQuery().getParameters(), false, null); - String[] intervals = getIntervals(dslObject.getQuery().getParameters().getIntervals()); - String whereOfTime = "__time >= DATE_FORMAT(FROM_UNIXTIME(UNIX_TIMESTAMP('".concat(intervals[0]).concat("')), '%Y-%m-%d %H:%i:%s')") - .concat("AND __time < DATE_FORMAT(FROM_UNIXTIME(UNIX_TIMESTAMP( '").concat(intervals[1]).concat("')), '%Y-%m-%d %H:%i:%s')"); - String granularity = dslObject.getQuery().getParameters().getGranularity(); - granularity = StringUtil.isEmpty(granularity) ? "PT15S" : granularity; - Period period = Period.parse(granularity); - Integer limit = dslObject.getQuery().getLimit(); - Integer offset = dslObject.getQuery().getOffset(); - String sql = String.format(Objects.requireNonNull(environment.getProperty("APP_DATA_SUMMARY")), - period.toStandardSeconds().getSeconds(), dslObject.getQuery().getDataSource(), whereOfTime, whereOfExactly, StringUtil.isEmpty(limit) ? "" : " LIMIT " + ((StringUtil.isEmpty(offset)) ? "" : offset + " ,") + limit); - Map<String, String> dataResult = executeQuery(sql); - if (String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(dataResult.get("status"))) { - return JSON.parseObject(dataResult.get("result"), BaseResult.class); - } else { - throw new BusinessException(dataResult.get("result")); - } - } - - @Override - public BaseResult buildInternalIPData(DSLObject dslObject) { - String whereOfExactly = getWhereFilter(dslObject.getQuery().getParameters(), true, null); - String[] intervals = getIntervals(dslObject.getQuery().getParameters().getIntervals()); - String whereOfTime = "recv_time >= UNIX_TIMESTAMP('" + intervals[0] + "') AND recv_time < UNIX_TIMESTAMP('" + intervals[1] + "')"; - Integer limit = dslObject.getQuery().getLimit(); - Integer offset = dslObject.getQuery().getOffset(); - String sql = String.format(Objects.requireNonNull(environment.getProperty("APP_INTERNAL_IP_SUMMARY")), - dslObject.getQuery().getDataSource(), whereOfTime, whereOfExactly, StringUtil.isEmpty(limit) ? "" : " LIMIT " + ((StringUtil.isEmpty(offset)) ? "" : offset + " ,") + limit); - Map<String, String> dataResult = executeQuery(sql); - if (String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(dataResult.get("status"))) { - return JSON.parseObject(dataResult.get("result"), BaseResult.class); - } else { - throw new BusinessException(dataResult.get("result")); - } - } - - @Override - public BaseResult buildAppDataRate(DSLObject dslObject) { - BaseResult baseResult; - DSLObject.QueryBean.Parameters parameters = dslObject.getQuery().getParameters(); - String whereOfExactly = getWhereFilter(parameters, false, null); - String whereOfTime = getWhereOfTime(parameters); - String sql = String.format(environment.getProperty("APP_DATA_RATE_SUMMARY"), - parameters.getGranularity(), - dslObject.getQuery().getDataSource(), - whereOfTime, - whereOfExactly, - parameters.getGranularity() - ); - Map<String, String> result = executeQuery(sql); - if (String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(result.get("status"))) { - Map resultMap = JSON.parseObject(result.get("result"), Map.class); - List<Map> data = (List<Map>) resultMap.get("data"); - baseResult = BaseResultGenerator.success("ok", data, (Map) resultMap.get("statistics")); - } else { - throw new BusinessException(result.get("result")); - } - return baseResult; - } - - @Override - public BaseResult buildAppTraffic(DSLObject dslObject) { - BaseResult baseResult; - DSLObject.QueryBean.Parameters parameters = dslObject.getQuery().getParameters(); - String whereOfExactly = getWhereFilter(parameters, false, null); - String whereOfTime = getWhereOfTime(parameters); - String sql = String.format(environment.getProperty("APP_TRAFFIC_SUMMARY"), - dslObject.getQuery().getDataSource(), - whereOfTime, - whereOfExactly - ); - Map<String, String> result = executeQuery(sql); - if (String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(result.get("status"))) { - Map resultMap = JSON.parseObject(result.get("result"), Map.class); - List<Map> data = (List<Map>) resultMap.get("data"); - baseResult = BaseResultGenerator.success("ok", data, (Map) resultMap.get("statistics")); - } else { - throw new BusinessException(result.get("result")); - } - return baseResult; - } - - /** - * 基于Name聚合，统计Protocol和Application在全栈下的流量 - * @param nodes - * @return - */ - private List<ProtocolTree> groupByName(List<ProtocolTree> nodes) { - Map<String, List<ProtocolTree>> resultMap = nodes.stream() - .collect(Collectors.groupingBy(ProtocolTree::getName, Collectors.collectingAndThen(Collectors.toList(), protocolTreeList -> { - - List<ProtocolTree> parentNodes = protocolTreeList.stream() - .filter(p -> StringUtil.isBlank(p.getParentId()) || !Splitter.on(".").omitEmptyStrings().splitToList(p.getParentId()).contains(p.getName())) - .collect(Collectors.toList()); - Map<String, Long> summedMetrics = Maps.newLinkedHashMap(); - Map<String, Long> totalBytesMetrics = Maps.newLinkedHashMap(); - totalBytesMetrics.put(Constants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES, 0L); - totalBytesMetrics.put(Constants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES, 0L); - - parentNodes.stream().filter(p -> !p.getName().equalsIgnoreCase(Constants.PROTOCOL_TREE_ROOT_NAME)).forEach(protocolTree -> { - totalBytesMetrics.put(Constants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES, totalBytesMetrics.get(Constants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES) + protocolTree.getSentBytes()); - totalBytesMetrics.put(Constants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES, totalBytesMetrics.get(Constants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES) + protocolTree.getReceivedBytes()); - for (String key : protocolTree.getMetrics().keySet()) { - String totalKey = StrUtil.upperFirstAndAddPre(key, "total"); - if (StringUtil.isEmpty(summedMetrics.get(totalKey))) { - summedMetrics.put(totalKey, Long.valueOf(protocolTree.getMetrics().get(key).toString())); - } else { - summedMetrics.put(totalKey, summedMetrics.get(totalKey) - + Long.parseLong(protocolTree.getMetrics().get(key).toString())); - } - } - }); - - protocolTreeList.stream().filter(p -> !p.getName().equalsIgnoreCase(Constants.PROTOCOL_TREE_ROOT_NAME)).forEach(protocolTree -> { - protocolTree.setTotalSentBytes(totalBytesMetrics.get(Constants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES)); - protocolTree.setTotalReceivedBytes(totalBytesMetrics.get(Constants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES)); - protocolTree.getMetrics().putAll(summedMetrics); - }); - - return protocolTreeList; - }))); - - List<ProtocolTree> protocolTreeList = Lists.newArrayList(); - resultMap.values().forEach(o -> { - protocolTreeList.addAll(o); - }); - - return protocolTreeList; - } -} diff --git a/src/main/java/com/mesalab/qgw/aspect/AuditLogAspect.java b/src/main/java/com/mesalab/qgw/aspect/AuditLogAspect.java index a25fa48a..ff562c8b 100644 --- a/src/main/java/com/mesalab/qgw/aspect/AuditLogAspect.java +++ b/src/main/java/com/mesalab/qgw/aspect/AuditLogAspect.java @@ -1,21 +1,21 @@ package com.mesalab.qgw.aspect; import cn.hutool.core.thread.ThreadUtil; -import cn.hutool.crypto.digest.DigestUtil; +import cn.hutool.core.util.StrUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.alibaba.fastjson2.JSON; import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.utils.QueryCacheUtils; -import com.mesalab.common.utils.HttpHelper; -import com.mesalab.qgw.model.basic.QueryProfile; +import com.mesalab.common.utils.MDCUtil; +import com.mesalab.qgw.model.basic.EngineConfigSource; +import com.mesalab.qgw.model.basic.SQLQueryContext; import com.mesalab.qgw.model.basic.AuditLog; import com.mesalab.qgw.model.basic.AuditServiceLog; import com.geedgenetworks.utils.IPUtil; -import com.geedgenetworks.utils.StringUtil; import org.aspectj.lang.JoinPoint; import org.aspectj.lang.annotation.*; import org.aspectj.lang.reflect.MethodSignature; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; @@ -23,7 +23,6 @@ import org.springframework.web.context.request.ServletRequestAttributes; import javax.servlet.http.HttpServletRequest; import java.lang.reflect.Method; import java.util.Arrays; -import java.util.Optional; @Aspect @@ -33,78 +32,79 @@ public class AuditLogAspect { private ThreadLocal<Long> startTime = ThreadUtil.createThreadLocal(false); private ThreadLocal<AuditServiceLog> auditLog = ThreadUtil.createThreadLocal(false); private static final Log log = LogFactory.get(); + private EngineConfigSource engineConfigSource; @Pointcut("@annotation(com.mesalab.qgw.model.basic.AuditLog)") - public void annotationPointCut(){} + public void annotationPointCut() { + } @Before("annotationPointCut()") public void doBefore(JoinPoint joinPoint) { - startTime.set(System.currentTimeMillis()); ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); - HttpServletRequest request = attributes.getRequest(); - String url = request.getRequestURL().toString(); - String method = request.getMethod(); - String remoteAddr = request.getRemoteAddr(); - String clientIp = IPUtil.getIpAddr(request); + AuditServiceLog auditServiceLog = new AuditServiceLog(); + String url = null; + if (attributes != null) { + HttpServletRequest request = attributes.getRequest(); + url = request.getRequestURL().toString(); + String method = request.getMethod(); + String remoteAddr = request.getRemoteAddr(); + String clientIp = IPUtil.getIpAddr(request); + auditServiceLog.setRemoteAddr(remoteAddr); + auditServiceLog.setClientIp(clientIp); + auditServiceLog.setMethod(method); + } String classMethod = joinPoint.getSignature().getDeclaringTypeName() + "." + joinPoint.getSignature().getName(); - Object paramObject = joinPoint.getArgs()[0]; - - MethodSignature sign = (MethodSignature)joinPoint.getSignature(); + MethodSignature sign = (MethodSignature) joinPoint.getSignature(); Method method2 = sign.getMethod(); AuditLog annotation = method2.getAnnotation(AuditLog.class); - AuditServiceLog auditServiceLog = new AuditServiceLog(); + auditServiceLog.setRequestParam(Arrays.toString(joinPoint.getArgs())); + auditServiceLog.setUrl(url); + auditServiceLog.setClassMethod(classMethod); auditServiceLog.setAnnotation(annotation.value()); - auditServiceLog.setRemoteAddr(remoteAddr); - auditServiceLog.setClientIp(clientIp); - auditServiceLog.setMethod(method); - if (log.isDebugEnabled()) { - auditServiceLog.setParam(Optional.ofNullable(paramObject).orElse("").toString()); - auditServiceLog.setUrl(url); - auditServiceLog.setClassMethod(classMethod); - } - auditServiceLog.setCacheKey(HttpHelper.generateCacheKey(request)); - auditServiceLog.setQueryKey(DigestUtil.md5Hex(StringUtil.createUUID())); + auditServiceLog.setRequestId(MDCUtil.getTraceId()); auditLog.set(auditServiceLog); } @AfterThrowing(value = "annotationPointCut()", throwing = "exception") - public void doAfterThrowingAdvice(JoinPoint joinPoint, Throwable exception){ - AuditServiceLog logAfter = auditLog.get(); - logAfter.setQueryKey(""); - logAfter.setParam(Arrays.toString(joinPoint.getArgs())); - log.error("Audit Log [error]:{}", JSON.toJSONString(logAfter)); + public void doAfterThrowingAdvice(JoinPoint joinPoint, Throwable exception) { + AuditServiceLog logAfter = auditLog.get(); + logAfter.setRequestParam(Arrays.toString(joinPoint.getArgs())); + log.error("Audit Log [error]: {}, reason {}", JSON.toJSONString(logAfter), exception.getMessage()); } @After(value = "annotationPointCut()") public void after(JoinPoint joinPoint) { - if (joinPoint.getArgs()[0] instanceof QueryProfile) { - QueryProfile param = (QueryProfile) joinPoint.getArgs()[0]; - auditLog.get().setDbType(param.getDbType()); - } + AuditServiceLog logAfter = auditLog.get(); + Object[] args = joinPoint.getArgs(); + if (args != null && args.length > 0 && args[0] instanceof SQLQueryContext) { + SQLQueryContext queryContext = (SQLQueryContext) joinPoint.getArgs()[0]; + auditLog.get().setDbEngine(queryContext.getDbEngine()); + } + long elapsed = System.currentTimeMillis() - startTime.get(); + logAfter.setElapsed(elapsed); + String logMessage = StrUtil.format("Audit Log [done]: {}", JSON.toJSONString(logAfter)); + log.info(logMessage); + if (elapsed > engineConfigSource.getHighLatencyThreshold()) { + log.warn(logMessage); + } } - - @AfterReturning(returning = "result", pointcut = "annotationPointCut()") public void doAfterReturning(Object result) { - AuditServiceLog logAfter = auditLog.get(); - if (StringUtil.isNotEmpty(result)) { - QueryCacheUtils.put(logAfter.getCacheKey(), result); - } - if(result instanceof BaseResult){ + AuditServiceLog logAfter = auditLog.get(); + if (result instanceof BaseResult) { BaseResult baseResult = (BaseResult) result; - baseResult.setQueryKey(logAfter.getQueryKey()); + baseResult.setRequestId(logAfter.getRequestId()); } - Long exeTime = System.currentTimeMillis() - startTime.get(); - logAfter.setExeTime(exeTime); - - log.info("Audit Log [completed]:{}", JSON.toJSONString(logAfter)); - + log.debug("Audit Log [success]: result {}", result); } - + @Autowired + private void setEngineConfigSource(EngineConfigSource engineConfigSource) { + this.engineConfigSource = engineConfigSource; + } } diff --git a/src/main/java/com/mesalab/qgw/benchmark/DialectWriter.java b/src/main/java/com/mesalab/qgw/benchmark/DialectWriter.java index 1f1fd839..6c6ec377 100644 --- a/src/main/java/com/mesalab/qgw/benchmark/DialectWriter.java +++ b/src/main/java/com/mesalab/qgw/benchmark/DialectWriter.java @@ -5,11 +5,10 @@ import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.file.FileWriter; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.fastjson2.JSON; import com.google.common.collect.Lists; -import com.mesalab.common.nacos.NacosConfig; import com.geedgenetworks.utils.DateUtils; -import com.mesalab.services.service.SQLDatasetService; +import com.mesalab.qgw.service.DatasetService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -17,7 +16,6 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -25,16 +23,10 @@ import java.util.Map; public class DialectWriter extends Writer { private static final Log log = LogFactory.get(); - private static final String EXECUTE_ENGINE= "execute_engine"; + private static final String BACKEND_ENGINE= "backend_engine"; private static final String TEMPLATE= "template"; - - @Autowired - private NacosConfig nacosConfig; - - @Autowired - private ConfigService systemConfigService; @Autowired - private SQLDatasetService sqlDatasetService; + DatasetService datasetService; @Override public String buildPocSQL(List<String> dialectList, String dialect, String category) { @@ -44,13 +36,13 @@ public class DialectWriter extends Writer { + File.separator + "benchmark" + File.separator + dialect + "_queries" + "_" + DateUtils.getCurrentDate(DateUtils.YYYYMMDD) + ".sql"; if (CollectionUtil.isEmpty(dialectList) && !Files.exists(Paths.get(path))) { - Map<String, Object> datasetResult = sqlDatasetService.getDatasets(Lists.newArrayList(), category, null); - ArrayList<Map<String, String>> datasetList = (ArrayList<Map<String, String>>) datasetResult.get("list"); + List<Map<String, Object>> datasetList = datasetService.getDatasets(Lists.newArrayList(), category, null); dialectList = Lists.newLinkedList(); if (CollectionUtil.isNotEmpty(datasetList)) { for (int i = 0; i < datasetList.size(); i++) { - if (datasetList.get(i).get(EXECUTE_ENGINE).equalsIgnoreCase(dialect)) { - dialectList.add(datasetList.get(i).get(TEMPLATE)); + if (String.valueOf(datasetList.get(i).get(BACKEND_ENGINE)).equalsIgnoreCase(dialect)) { + Map map = JSON.parseObject(String.valueOf(datasetList.get(i).get(TEMPLATE)), Map.class); + dialectList.add(String.valueOf(map.get("statement"))); } } } diff --git a/src/main/java/com/mesalab/qgw/benchmark/Writer.java b/src/main/java/com/mesalab/qgw/benchmark/Writer.java index 63a37ba7..c0298307 100644 --- a/src/main/java/com/mesalab/qgw/benchmark/Writer.java +++ b/src/main/java/com/mesalab/qgw/benchmark/Writer.java @@ -1,7 +1,6 @@ package com.mesalab.qgw.benchmark; import cn.hutool.core.io.IORuntimeException; -import cn.hutool.core.io.file.FileReader; import cn.hutool.core.io.file.FileWriter; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; @@ -14,15 +13,14 @@ import com.jayway.jsonpath.JsonPathException; import com.mesalab.common.nacos.NacosConfig; import com.mesalab.common.nacos.NacosConst; import com.geedgenetworks.utils.StringUtil; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.services.service.SQLDatasetService; +import com.mesalab.qgw.service.DatabaseService; +import com.mesalab.qgw.service.DatasetService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -33,38 +31,12 @@ public abstract class Writer { @Autowired NacosConfig nacosConfig; @Autowired - MetadataService metadataService; + DatabaseService databaseService; @Autowired - SQLDatasetService sqlDatasetService; + DatasetService datasetService; private static final Log log = LogFactory.get(); - public static final boolean IS_TITLE = false; - public static final String TITLE_PREFIX = "--Q"; - public static final String LABEL_COMMON_FILTER = "@common_filter"; - public static final String LABEL_INDEX_FILTER = "@index_filter"; - public static final String LABEL_TIME_START = "@start"; - public static final String LABEL_TIME_END = "@end"; - public Map<String, List<String>> getFilterOptions(String jsonString) { - Map<String, List<String>> environment = Maps.newLinkedHashMap(); - try { - List<String> startTime = JsonPath.read(jsonString, "$.filters[?(@.name=='@start')].value"); - List<String> endTime = JsonPath.read(jsonString, "$.filters[?(@.name=='@end')].value"); - environment.put(LABEL_TIME_START, startTime); - environment.put(LABEL_TIME_END, endTime); - List<List<String>> commonFilter = JsonPath.read(jsonString, "$.filters[?(@.name=='@common_filter')].value"); - List<List<String>> indexFilter = JsonPath.read(jsonString, "$.filters[?(@.name=='@index_filter')].value"); - if (StringUtil.isNotEmpty(commonFilter)) { - environment.put(LABEL_COMMON_FILTER, commonFilter.stream().findFirst().get()); - } - if (StringUtil.isNotEmpty(indexFilter)) { - environment.put(LABEL_INDEX_FILTER, indexFilter.stream().findFirst().get()); - } - } catch (JsonPathException e) { - log.error(" Poc SQL is not found {} ", e); - } - return environment; - } protected String generatePocSQL(List<String> list, FileWriter writer) { String pocSQL = ""; @@ -73,14 +45,14 @@ public abstract class Writer { writer.getFile().createNewFile(); } List<LinkedHashMap> variables = Lists.newArrayList(); - Object codeInfo = metadataService.getCfg(NacosConst.SQL_DATASETS_VARIABLES); + Object codeInfo = databaseService.getCfg(NacosConst.DATASETS_VARIABLES); if (StringUtil.isNotEmpty(codeInfo)) { Object json = JSON.toJSON(codeInfo); - variables = (List<LinkedHashMap>) JSON.parseArray(json.toString(), LinkedHashMap.class); + variables = JSON.parseArray(json.toString(), LinkedHashMap.class); } List<String> resultLines = Lists.newArrayList(); - for (String line : list){ - String sql = sqlDatasetService.buildExecSQL(variables, line); + for (String line : list) { + String sql = datasetService.buildExecSQL(variables, line); resultLines.add(sql); } writer.appendLines(resultLines); @@ -92,52 +64,6 @@ public abstract class Writer { return pocSQL; } - - @Value("${nacos.config.server-addr}") - private String nacosServerAddr; - - protected String buildSnapshotDataFilePath(String dataId) { - StringBuilder filePath = new StringBuilder(buildPrefixPath()); - filePath.append(NacosConst.SNAPSHOT_TENANT); - filePath.append(File.separator); - filePath.append(nacosConfig.getNamespace()); - filePath.append(File.separator); - filePath.append(nacosConfig.getGroup()); - filePath.append(File.separator); - filePath.append(dataId); - log.info("dataId:{} snapshot path:{}", dataId, filePath.toString()); - return filePath.toString(); - } - - protected String buildLocalDataFilePath(String dataId) { - StringBuilder filePath = new StringBuilder(buildPrefixPath()); - filePath.append(NacosConst.NACOS_LOCAL_DATA); - filePath.append(File.separator); - filePath.append(NacosConst.CONFIG_DATA_TENANT); - filePath.append(File.separator); - filePath.append(nacosConfig.getNamespace()); - filePath.append(File.separator); - filePath.append(nacosConfig.getGroup()); - filePath.append(File.separator); - filePath.append(dataId); - log.info("dataId:{} local data path:{}", dataId, filePath.toString()); - return filePath.toString(); - } - - private String buildPrefixPath() { - StringBuilder filePath = new StringBuilder(LocalConfigInfoProcessor.LOCAL_SNAPSHOT_PATH); - filePath.append(File.separator); - filePath.append(NacosConst.FIXED); - filePath.append(NacosConst.RAMPANT); - nacosServerAddr = nacosServerAddr.replaceAll(NacosConst.COLON, NacosConst.UNDERLINE); - filePath.append(nacosServerAddr).append(NacosConst.RAMPANT); - filePath.append(nacosConfig.getNamespace()); - filePath.append(NacosConst.UNDERLINE); - filePath.append(NacosConst.NACOS); - filePath.append(File.separator); - return filePath.toString(); - } - protected abstract String buildPocSQL(List<String> dialectList, String dialect, String category); diff --git a/src/main/java/com/mesalab/qgw/constant/DataTypeConst.java b/src/main/java/com/mesalab/qgw/constant/DataTypeConst.java new file mode 100644 index 00000000..bc3e1b62 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/constant/DataTypeConst.java @@ -0,0 +1,23 @@ +package com.mesalab.qgw.constant; + +/** + * TODO + * + * @Classname DataTypeConst + * @Date 2024/3/4 10:22 + * @Author wWei + */ +public class DataTypeConst { + public static final String TIMESTAMP = "timestamp"; + public static final String TIMESTAMP_FORMAT = "timestamp_format"; + public static final String TIMESTAMP_MS_FORMAT = "timestamp_ms_format"; + public static final String UNIX_TIMESTAMP = "unix_timestamp"; + public static final String UNIX_TIMESTAMP_MS = "unix_timestamp_ms"; + public static final String DATE_TIME_64 = "datetime64"; + public static final String DATE = "date"; + public static final String IP = "ip"; + public static final String DOMAIN = "domain"; + public static final String FQDN = "fqdn"; + public static final String APP = "app"; + public static final String HDR_HISTOGRAM = "hdr_histogram"; +} diff --git a/src/main/java/com/mesalab/qgw/constant/DslIdentifierNameConst.java b/src/main/java/com/mesalab/qgw/constant/DslIdentifierNameConst.java index cb2f6dd2..90115236 100644 --- a/src/main/java/com/mesalab/qgw/constant/DslIdentifierNameConst.java +++ b/src/main/java/com/mesalab/qgw/constant/DslIdentifierNameConst.java @@ -1,5 +1,7 @@ package com.mesalab.qgw.constant; +import java.util.*; + /** * TODO * @@ -15,6 +17,34 @@ public class DslIdentifierNameConst { public static final String APPLICATION_AND_PROTOCOL_APP_RELATED_INTERNAL_IPS = "application-and-protocol-app-related-internal-ips"; public static final String APPLICATION_AND_PROTOCOL_APP_THROUGHPUT = "application-and-protocol-app-throughput"; public static final String APPLICATION_AND_PROTOCOL_APP_SUMMARY = "application-and-protocol-app-summary"; - public static final String REAL_TIME_DATA_ANALYTICS_SUBSCRIBER_ID_RELATE_IP = "real-time-data-analytics-subscriber-id-relate-ip"; - public static final String REAL_TIME_DATA_ANALYTICS_MOBILE_IDENTITY_RELATE_TEID = "real-time-data-analytics-mobile-identity-relate-teid"; + public static final String IP_LEARNING_FQDN_RELATE_IP = "ip-learning-fqdn-relate-ip"; + public static final String IP_LEARNING_ACTIVE_IP = "ip-learning-active-ip"; + public static final String FIELD_DISCOVERY = "field_discovery"; + public static final String DATAPATH_PACKET_COMBINE = "datapath_telemetry_packet_combine"; + public static final String TRAFFIC_SPECTRUM_SUMMARY = "traffic-spectrum-summary"; + public static final String TRAFFIC_SPECTRUM_UNIQUE_CLIENT_AND_SERVER_IPS = "traffic-spectrum-unique-client-and-server-ips"; + public static final String TRAFFIC_SPECTRUM_APP_DISTRIBUTION = "traffic-spectrum-app-distribution"; + public static final String TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE = "traffic-spectrum-client-ip-connect-application-usage"; + public static final String TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND = "traffic-spectrum-network-throughput-trend"; + public static final Map<String, Set<String>> IDENTIFIER_NAME_SOURCE_MAPPING; + + static { + IDENTIFIER_NAME_SOURCE_MAPPING = new HashMap<>(); + IDENTIFIER_NAME_SOURCE_MAPPING.put(APPLICATION_AND_PROTOCOL_SUMMARY, new HashSet<>(List.of("application_protocol_stat"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(APPLICATION_AND_PROTOCOL_TREE_COMPOSITION, new HashSet<>(List.of("application_protocol_stat"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(APPLICATION_AND_PROTOCOL_TREE_THROUGHPUT, new HashSet<>(List.of("application_protocol_stat"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(APPLICATION_AND_PROTOCOL_TOP_APPS, new HashSet<>(List.of("application_protocol_stat"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(APPLICATION_AND_PROTOCOL_APP_RELATED_INTERNAL_IPS, new HashSet<>(List.of("session_record"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(APPLICATION_AND_PROTOCOL_APP_THROUGHPUT, new HashSet<>(List.of("application_protocol_stat"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(APPLICATION_AND_PROTOCOL_APP_SUMMARY, new HashSet<>(List.of("application_protocol_stat"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(IP_LEARNING_FQDN_RELATE_IP, new HashSet<>()); + IDENTIFIER_NAME_SOURCE_MAPPING.put(IP_LEARNING_ACTIVE_IP, new HashSet<>()); + IDENTIFIER_NAME_SOURCE_MAPPING.put(FIELD_DISCOVERY, new HashSet<>()); + IDENTIFIER_NAME_SOURCE_MAPPING.put(DATAPATH_PACKET_COMBINE, new HashSet<>(List.of("datapath_telemetry_record"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(TRAFFIC_SPECTRUM_SUMMARY, new HashSet<>(List.of("traffic_sketch_metric"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(TRAFFIC_SPECTRUM_UNIQUE_CLIENT_AND_SERVER_IPS, new HashSet<>(List.of("traffic_sketch_metric"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(TRAFFIC_SPECTRUM_APP_DISTRIBUTION, new HashSet<>(List.of("traffic_sketch_metric"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE, new HashSet<>(List.of("traffic_sketch_metric"))); + IDENTIFIER_NAME_SOURCE_MAPPING.put(TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND, new HashSet<>(List.of("traffic_sketch_metric"))); + } } diff --git a/src/main/java/com/mesalab/qgw/constant/ExampleDataModeConst.java b/src/main/java/com/mesalab/qgw/constant/ExampleDataModeConst.java new file mode 100644 index 00000000..7985e003 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/constant/ExampleDataModeConst.java @@ -0,0 +1,14 @@ +package com.mesalab.qgw.constant; + +/** + * TODO + * + * @Classname ExampleDataModeConst + * @Date 2024/3/8 13:44 + * @Author wWei + */ +public class ExampleDataModeConst { + public static final String RANGE = "range"; + public static final String ENUM = "enum"; + public static final String SEQUENCE = "sequence"; +} diff --git a/src/main/java/com/mesalab/qgw/constant/MetaConst.java b/src/main/java/com/mesalab/qgw/constant/MetaConst.java new file mode 100644 index 00000000..2f3f964a --- /dev/null +++ b/src/main/java/com/mesalab/qgw/constant/MetaConst.java @@ -0,0 +1,18 @@ +package com.mesalab.qgw.constant; + +/** + * TODO + * + * @Classname MetaConst + * @Date 2024/3/7 14:29 + * @Author wWei + */ +public class MetaConst { + public static final String META_CATEGORY = "category"; + public static final String META_NAME = "name"; + public static final String META_TYPE = "type"; + public static final String META_DATA_TYPE = "data_type"; + public static final String META_FIELD_NAME = "field_name"; + public static final String META_CATEGORY_METRIC = "Metric"; + public static final String META_CATEGORY_DIMENSION = "Dimension"; +} diff --git a/src/main/java/com/mesalab/qgw/constant/QGWMessageConst.java b/src/main/java/com/mesalab/qgw/constant/QGWMessageConst.java index c9aa22f0..6e89a909 100644 --- a/src/main/java/com/mesalab/qgw/constant/QGWMessageConst.java +++ b/src/main/java/com/mesalab/qgw/constant/QGWMessageConst.java @@ -22,9 +22,11 @@ public class QGWMessageConst { public static final String SQL_PARSE_ERROR_NOT_FOUNT_TABLE = "unable to parse table name."; - public static final String DIAGNOSIS_OPTION_ERROR = "not support option value."; + public static final String CONSISTENCY_OPTION_ERROR = "not support consistency option value."; + + public static final String BENCHMARK_TEST_PARAM_CANNOT_BE_EMPTY = "The benchmark test param value cannot be empty."; - public static final String DIAGNOSIS_CATEGORY_ERROR = "not support category value."; + public static final String DIAGNOSIS_OPTION_ERROR = "not support option value."; public static final String NOT_SUPPORT_DML_PARSER = "not support DML Parser."; @@ -110,5 +112,26 @@ public class QGWMessageConst { public static final String QUERY_CUSTOM_FIELD_DISCOVERY_ERROR = "The param custom.field_discovery.* illegal."; + public static final String DATASET_BACKEND_ENGINE_ERROR = "The dataset backend_engine is not supported, backend_engine value must be :[ qgw| clickhouse| druid| hbase]"; + + public static final String BUCKET_NAME_ERROR = "The bucket_name cannot be empty."; + + public static final String DATASET_ID_NOT_EXIST = "The dataset identifier_name is not exist."; + + public static final String KB_FILE_ID_CANNOT_BE_EMPTY = "The knowledge base file id cannot be empty."; + + public static final String KB_FILE_NAME_CANNOT_BE_EMPTY = "The knowledge base file name cannot be empty."; + + public static final String KB_FILE_VERSION_CANNOT_BE_EMPTY = "The knowledge base file version cannot be empty."; + + public static final String KB_FILE_IS_VALID_CANNOT_BE_EMPTY = "The knowledge base file is_valid cannot be empty."; + + public static final String KB_FILE_FORMAT_CANNOT_BE_EMPTY = "The knowledge base file format cannot be empty."; + + public static final String KB_FILE_FORMAT_NOT_SUPPORT = "The knowledge base file format is not supported, format value must be :[ MMDB| CSV| TXT| AES]"; + + public static final String SCHEMA_FUNCTION_CONSTRAINT_ERROR ="The %s schema %s constraint functions has %s, does not exit in the list of supported aggregate functions."; + + public static final String SCHEMA_FIELD_TYPE_NEED_CHECK = "The %s schema fields type need to be checked: %s."; } diff --git a/src/main/java/com/mesalab/network/common/Constants.java b/src/main/java/com/mesalab/qgw/constant/dsl/LiveChartConstants.java index db7c632a..fd445c41 100644 --- a/src/main/java/com/mesalab/network/common/Constants.java +++ b/src/main/java/com/mesalab/qgw/constant/dsl/LiveChartConstants.java @@ -1,27 +1,27 @@ -package com.mesalab.network.common; +package com.mesalab.qgw.constant.dsl; -public class Constants { +/** + * @author wangwei + */ +public class LiveChartConstants { public static final String PROTOCOL_TREE_ROOT_NAME = "Protocols"; public static final String ENCAPSULATION_PATH_SEPARATOR = "."; public static final String PROTOCOL_ETHERNET_NODE = "ETHERNET"; public static final String PROTOCOLS_PARENT_ID = "ROOT"; - public static final String PROTOCOL_TREE_METRIC_ENCAPSULATION_PATHS = "encapsulationPaths"; - public static final String PROTOCOL_TREE_METRIC_PROTOCOLS_AND_APPLICATIONS = "protocolsAndApplications"; + public static final String PROTOCOL_TREE_METRIC_ENCAPSULATION_PATHS = "encapsulation_paths"; + public static final String PROTOCOL_TREE_METRIC_PROTOCOLS_AND_APPLICATIONS = "protocols_and_applications"; public static final String PROTOCOL_TREE_METRIC_SESSIONS = "sessions"; public static final String PROTOCOL_TREE_METRIC_C2S_BYTES = "c2s_bytes"; public static final String PROTOCOL_TREE_METRIC_S2C_BYTES = "s2c_bytes"; public static final String PROTOCOL_TREE_METRIC_C2S_PKTS = "c2s_pkts"; public static final String PROTOCOL_TREE_METRIC_S2C_PKTS = "s2c_pkts"; - public static final String PROTOCOL_TREE_METRIC_SENT_PACKETS = "sentPackets"; - public static final String PROTOCOL_TREE_METRIC_RECEIVED_PACKETS = "receivedPackets"; - public static final String PROTOCOL_TREE_METRIC_TOTAL_SENT_PACKETS = "totalSentPackets"; - public static final String PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_PACKETS = "totalReceivedPackets"; - public static final String PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES = "totalSentBytes"; - public static final String PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES = "totalReceivedBytes"; + public static final String PROTOCOL_TREE_METRIC_SENT_PACKETS = "sent_packets"; + public static final String PROTOCOL_TREE_METRIC_RECEIVED_PACKETS = "received_packets"; + public static final String PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES = "total_sent_bytes"; + public static final String PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES = "total_received_bytes"; - public static final String NETWORK_OVERVIEW_METRIC_UNIQ_CLIENT_IP = "uniq_client_ip"; public static final String NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS = "total_sessions"; public static final String NETWORK_OVERVIEW_METRIC_TOTAL_BYTES = "total_bytes"; public static final String NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS = "total_packets"; @@ -35,8 +35,6 @@ public class Constants { public static final String NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PERCENT = "tcp_retransmissions_percent"; public static final String NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS = "fragmentation_packets"; public static final String NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PERCENT = "fragmentation_percent"; - - public static final String NETWORK_OVERVIEW_METRIC_DATA_RATE = "data_rate"; } diff --git a/src/main/java/com/mesalab/qgw/constant/dsl/TrafficSpectrumConstants.java b/src/main/java/com/mesalab/qgw/constant/dsl/TrafficSpectrumConstants.java new file mode 100644 index 00000000..fe19d138 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/constant/dsl/TrafficSpectrumConstants.java @@ -0,0 +1,46 @@ +package com.mesalab.qgw.constant.dsl; + +/** + * TODO + * + * @Classname TrafficSpectrumConstants + * @Date 2024/5/23 16:19 + * @Author wWei + */ +public class TrafficSpectrumConstants { + public static final String SOURCE = "source"; + public static final String TARGET = "target"; + public static final String DIRECTION = "direction"; + public static final String SESSIONS = "sessions"; + public static final String BYTES = "bytes"; + public static final String PACKETS = "packets"; + public static final String UNKNOWN_APP_BYTES = "unknown_app_bytes"; + public static final String UNCATEGORIZED_PERCENT = "uncategorized_percent"; + public static final String ASYMMETRIC_FLOWS = "asymmetric_flows"; + public static final String ASYMMETRIC_FLOWS_PERCENT = "asymmetric_flows_percent"; + public static final String FRAGMENTATION_PACKETS = "fragmentation_packets"; + public static final String FRAGMENTATION_PERCENT = "fragmentation_percent"; + public static final String INTERNAL_UNIQ_CLIENT_IP = "internal_uniq_client_ip"; + public static final String INTERNAL_UNIQ_SERVER_IP = "internal_uniq_server_ip"; + public static final String EXTERNAL_UNIQ_CLIENT_IP = "external_uniq_client_ip"; + public static final String EXTERNAL_UNIQ_SERVER_IP = "external_uniq_server_ip"; + public static final String TOP_SERVER_IPS = "top_server_ips"; + public static final String TOP_SERVER_DOMAINS = "top_server_domains"; + public static final String MAX_RATE = "max_rate"; + public static final String AVG_RATE = "avg_rate"; + public static final String INTERNAL_NODE = "internal_node"; + public static final String EXTERNAL_NODE = "external_node"; + public static final String NODES = "nodes"; + public static final String LINKS = "links"; + public static final String OBJECT_TYPE = "object_type"; + public static final String CATEGORY = "category"; + public static final String NAME = "name"; + public static final String ID = "id"; + public static final String OTHER = "Other"; + public static final String OBJECT_TYPE_APPLICATION = "application"; + public static final String OBJECT_TYPE_IP = "ip"; + public static final String PRIVATE_IP = "Private IP"; + public static final String OUTBOUND = "Outbound"; + public static final String INBOUND = "Inbound"; + public static final String SPLIT_HYPHEN = "-"; +} diff --git a/src/main/java/com/mesalab/qgw/controller/DatabaseController.java b/src/main/java/com/mesalab/qgw/controller/DatabaseController.java new file mode 100644 index 00000000..66de5887 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/controller/DatabaseController.java @@ -0,0 +1,153 @@ +package com.mesalab.qgw.controller; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.NumberUtil; +import com.alibaba.fastjson2.JSON; +import com.clearspring.analytics.util.Lists; +import com.google.common.collect.Sets; +import com.jayway.jsonpath.JsonPath; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.enums.MetadataType; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.AuditLog; +import com.mesalab.qgw.model.job.StorageDeletionInfo; +import com.mesalab.qgw.service.DatabaseService; +import com.geedgenetworks.utils.StringUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +@RestController +@RequestMapping(value = "v1/database") +public class DatabaseController { + + @Autowired + private DatabaseService databaseService; + + private final static HashSet<String> VISIBILITY = Sets.newHashSet("enabled", "hidden", "disabled"); + + + @RequestMapping(value = "/table/{table_name}/schema", consumes = "application/x-www-form-urlencoded") + @AuditLog("DatabaseController.getSchemaByTable") + public BaseResult getSchemaByTable(@PathVariable("table_name") String tableName) { + log.debug("SCHEMA信息获取,参数为{}", tableName); + + if (StringUtil.isBlank(tableName)) { + return BaseResultGenerator.success4Message("ok"); + } + return BaseResultGenerator.success(databaseService.getSchemaInfo(MetadataType.FIELDS.getValue(), tableName, true)); + } + + @PutMapping(value = "/table/{table_name}/schema", consumes = "application/json") + @AuditLog("DatabaseController.updateSchema") + public BaseResult updateSchema(@PathVariable("table_name") String tableName, @RequestBody Map<String, Object> obj) { + log.info("update Schema api, params is: {}-{}", tableName, obj); + if (StringUtil.isBlank(tableName) || StringUtil.isEmpty(obj)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_PARAM_ERROR)); + } + validationParam(tableName, obj); + return databaseService.updateSchema(tableName, obj); + } + + @RequestMapping(value = "/{dbName}/table", consumes = "application/x-www-form-urlencoded") + @AuditLog("DatabaseController.getTableByDB") + public BaseResult getTableByDB(@PathVariable String dbName) { + log.debug("SCHEMA信息获取,参数为{}", dbName); + + if (StringUtil.isBlank(dbName)) { + + return BaseResultGenerator.success4Message("ok"); + } + return BaseResultGenerator.success(databaseService.getSchemaInfo(MetadataType.TABLES.getValue(), dbName, true)); + } + + @GetMapping(value = "/storage/quota", consumes = "application/x-www-form-urlencoded") + @AuditLog("DatabaseController.storageQuota") + public BaseResult storageQuota() { + return databaseService.getStorageQuota(); + } + + @PutMapping(value = "/storage/quota", consumes = "application/json") + @AuditLog("DatabaseController.storageSetting") + public BaseResult storageSetting(@RequestBody List<StorageDeletionInfo> list) { + log.warn("数据配额设置, 参数: params is {}", list); + if (StringUtil.isEmpty(list)) { + return BaseResultGenerator.failure(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.PARAMETER_NOT_OBTAINED)); + } + for (StorageDeletionInfo info : list) { + if (StringUtil.isBlank(info.getType()) || StringUtil.isBlank(String.valueOf(info.getMaxDays()))) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.LOGTYPE_AND_MAXDAYS_NOT_NULL)); + } + if ("ALL".equalsIgnoreCase(info.getType()) && list.size() != 1) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.CANNOT_COEXIST_OTHER_TYPES)); + } + if (!(info.getMaxDays() >= 0 && info.getMaxDays() <= 2000)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.MAXDAYS_ERROR)); + } + } + return databaseService.deleteStorage(list); + } + + @RequestMapping(value = "/storage/quota", method = RequestMethod.DELETE, consumes = "application/x-www-form-urlencoded") + @AuditLog("DatabaseController.storageDeletion") + public BaseResult storageDeletion(@RequestParam String type) { + List<StorageDeletionInfo> deletionInfoList = Lists.newArrayList(); + StorageDeletionInfo deletionInfo = new StorageDeletionInfo(); + deletionInfo.setType(type); + deletionInfo.setMaxDays(0); + deletionInfoList.add(deletionInfo); + return databaseService.deleteStorage(deletionInfoList); + } + + @RequestMapping(value = "/storage/quota/daily_usage", consumes = "application/x-www-form-urlencoded") + @AuditLog("DatabaseController.dailyTrendOfStorage") + public BaseResult dailyTrendOfStorage(@RequestParam Map<String, Object> param) { + String searchStartTime = StringUtil.stripToEmpty((String) param.get("start_time")); + String searchEndTime = StringUtil.stripToEmpty((String) param.get("end_time")); + return databaseService.dailyTrendOfStorage(searchStartTime, searchEndTime); + } + + private void validationParam(String name, Map<String, Object> obj) { + if (!databaseService.getAllTable().contains(name)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), name + QGWMessageConst.NOT_SUPPORT_SETTING_TTL)); + } + String param = JSON.toJSONString(obj); + if (!VISIBILITY.containsAll(JsonPath.read(param, "$.fields[?(@.doc.visibility != null)].doc.visibility"))) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_ERROR_VISIBILITY_VALUE)); + } + List<Object> schemaDocTTL = JsonPath.read(param, "$.[?(@.doc.ttl != null)].doc.ttl"); + if (StringUtil.isNotEmpty(schemaDocTTL.stream().filter(o -> !(NumberUtil.isLong(String.valueOf(o)))).toArray())) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_ERROR_TTL_VALUE)); + } + List<Object> fieldDocTTL = JsonPath.read(param, "$.fields[?(@.doc.ttl != null)].doc.ttl"); + Object[] objects3 = fieldDocTTL.parallelStream().filter(o -> !NumberUtil.isLong(String.valueOf(o))).toArray(); + if (StringUtil.isNotEmpty(objects3)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_ERROR_TTL_VALUE)); + } + if (schemaDocTTL.size() > 0 && fieldDocTTL.size() > 0 + && CollectionUtil.max(schemaDocTTL.stream().map(o -> Long.parseLong(o.toString())).collect(Collectors.toList())) < CollectionUtil.max(fieldDocTTL.stream().map(o -> Long.parseLong(o.toString())).collect(Collectors.toList()))) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_ERROR_TTL_SIZE)); + } + } +} diff --git a/src/main/java/com/mesalab/qgw/controller/DatasetController.java b/src/main/java/com/mesalab/qgw/controller/DatasetController.java new file mode 100644 index 00000000..7fc351ac --- /dev/null +++ b/src/main/java/com/mesalab/qgw/controller/DatasetController.java @@ -0,0 +1,62 @@ +package com.mesalab.qgw.controller; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.http.HttpStatus; +import com.google.common.collect.Lists; +import com.mesalab.common.entity.BaseResult; + +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.enums.DBEngineType; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.AuditLog; +import com.mesalab.qgw.service.DatasetService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import java.util.*; + +@Slf4j +@RestController +@RequestMapping(value = "/v1/dataset") +public class DatasetController { + + @Autowired + private DatasetService datasetService; + + @GetMapping(value = "/{identifier_name}") + @AuditLog("DatasetController.getDataset") + public BaseResult getDataset(@PathVariable("identifier_name") String identifierName) { + log.info("Get Dataset, ID is: {}", identifierName); + Map<String, Object> dataset = datasetService.getDataset(identifierName); + return BaseResultGenerator.success(Lists.newArrayList(dataset)); + } + + @GetMapping + @AuditLog("DatasetController.getDatasets") + public BaseResult getDatasets(@RequestParam(value = "identifier_names", required = false) String datasetIds, @RequestParam(required = false) String category, @RequestParam(value = "backend_engine", required = false) String backendEngine) { + if (StrUtil.isNotBlank(backendEngine)) { + if (Arrays.stream(DBEngineType.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(backendEngine))) { + throw new QGWBusinessException(HttpStatus.HTTP_BAD_REQUEST, CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.DATASET_BACKEND_ENGINE_ERROR)); + } + } + log.info("Get Datasets. IDs is: {}, category is: {}, backendEngine is: {}", datasetIds, category, backendEngine); + return BaseResultGenerator.success(datasetService.getDatasets(Arrays.asList(StrUtil.split(datasetIds, ",")), category, backendEngine)); + } + + @GetMapping(value = "/{identifier_name}/result_preview") + @AuditLog("DatasetController.getPreview") + public BaseResult getPreview(@PathVariable("identifier_name") String identifierName) { + return datasetService.getPreview(identifierName); + } + + @GetMapping(value = "/global_variable") + @AuditLog("DatasetController.getVariable") + public BaseResult getVariable() { + return BaseResultGenerator.success(datasetService.getVariable()); + } +}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/qgw/controller/DiagnosisController.java b/src/main/java/com/mesalab/qgw/controller/DiagnosisController.java index 896a0287..b6028ed1 100644 --- a/src/main/java/com/mesalab/qgw/controller/DiagnosisController.java +++ b/src/main/java/com/mesalab/qgw/controller/DiagnosisController.java @@ -1,22 +1,15 @@ package com.mesalab.qgw.controller; -import cn.hutool.core.util.StrUtil; import cn.hutool.http.HttpStatus; import com.alibaba.fastjson.JSONObject; -import com.fasterxml.jackson.annotation.JsonProperty; import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.utils.StringUtil; +import com.mesalab.common.enums.DBEngineType; import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.common.enums.DiagnosisOptionEnum; -import com.mesalab.common.exception.BusinessException; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.AuditLog; import com.mesalab.qgw.service.DiagnosisService; -import com.mesalab.services.common.enums.EntityQueryType; import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.expression.StringValue; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; @@ -34,51 +27,26 @@ public class DiagnosisController { private DiagnosisService diagnosisService; @RequestMapping("/component_info") + @AuditLog("DiagnosisController.versionInfo") public JSONObject versionInfo() { return diagnosisService.getVersionInfo(); } - @RequestMapping(value = "ttl_consistency_check", method = RequestMethod.POST) - public BaseResult monitorCluster() { - log.info("consistency check interface."); - return diagnosisService.consistencyCheck(); - } @RequestMapping(value = "/schema_check") + @AuditLog("DiagnosisController.validateSchema") public BaseResult validateSchema() { return diagnosisService.validateSchema(); } - @RequestMapping(value = "/metadata_check") - public BaseResult validateMetadata() { - return diagnosisService.validateMetadata(); - } - - @RequestMapping(value = "/sql_benchmark") - public BaseResult executePocSql(@RequestParam(value = "is_saved",required = false) @JsonProperty("isSaved") String isSaved, @RequestParam(defaultValue = "validation") String option, @RequestParam(required = false) String category) throws BusinessException { - if (Arrays.stream(DiagnosisOptionEnum.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(option))) { - throw new QGWBusinessException(HttpStatus.HTTP_BAD_REQUEST, ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.DIAGNOSIS_OPTION_ERROR)); - } - return diagnosisService.runPocSQL(Boolean.parseBoolean(isSaved), option, category); - } - @RequestMapping({"/sql", "/sql/{dialect}/{queryNo}"}) + @AuditLog("DiagnosisController.getPocSql") public BaseResult getPocSql(@PathVariable String dialect, @PathVariable(required = false) Integer queryNo) { - if (Arrays.stream(DBTypeEnum.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(dialect))){ - throw new QGWBusinessException(HttpStatus.HTTP_BAD_REQUEST, ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.DIALECT_VALUE_ERROR)); + if (Arrays.stream(DBEngineType.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(dialect))){ + throw new QGWBusinessException(HttpStatus.HTTP_BAD_REQUEST, CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.DIALECT_VALUE_ERROR)); } return diagnosisService.getPocSQL(dialect, queryNo); } - @RequestMapping(value = "/entity") - public BaseResult getMessageInfo(String option) { - log.info("Entity recommendation proportion statistical information interface, param is: {}", option); - if (!EntityQueryType.TOPSERVERIP.getType().equalsIgnoreCase(option) && !EntityQueryType.TOPSNI.getType().equalsIgnoreCase(option)) { - throw new QGWBusinessException(HttpStatus.HTTP_BAD_REQUEST, ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.DIAGNOSIS_OPTION_ERROR)); - } - return diagnosisService.getMessageInfo(option); - } } diff --git a/src/main/java/com/mesalab/qgw/controller/DslController.java b/src/main/java/com/mesalab/qgw/controller/DslController.java deleted file mode 100644 index 094c1875..00000000 --- a/src/main/java/com/mesalab/qgw/controller/DslController.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.mesalab.qgw.controller; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.qgw.constant.DslIdentifierNameConst; -import com.mesalab.qgw.model.basic.DSLProfile; -import com.mesalab.qgw.service.DslService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - - -/** - * TODO - * - * @Classname DslController - * @Date 2023/11/25 17:39 - * @Author wWei - */ -@RestController -@RequestMapping(value = "/dsl") -public class DslController { - - private static final Log log = LogFactory.get(); - - @Autowired - DslService dslService; - - @PostMapping(consumes = "application/json") - public BaseResult post(@RequestBody DSLProfile dslProfile) { - log.info("HTTP REST DSL, params is: {}", dslProfile); - if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_SUMMARY.equals(dslProfile.getName())) { - return dslService.appAndProtocolSummary(dslProfile); - } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_TREE_COMPOSITION.equals(dslProfile.getName())) { - return dslService.appAndProtocolTreeComposition(dslProfile); - } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_TREE_THROUGHPUT.equals(dslProfile.getName())) { - return dslService.applicationAndProtocolTreeThroughput(dslProfile); - } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_TOP_APPS.equals(dslProfile.getName())) { - return dslService.applicationAndProtocolTopApp(dslProfile); - } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_APP_RELATED_INTERNAL_IPS.equals(dslProfile.getName())) { - return dslService.applicationAndProtocolAppRelatedInternalIps(dslProfile); - } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_APP_THROUGHPUT.equals(dslProfile.getName())) { - return dslService.applicationAndProtocolAppThroughput(dslProfile); - } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_APP_SUMMARY.equals(dslProfile.getName())) { - return dslService.applicationAndProtocolAppSummary(dslProfile); - } else if (DslIdentifierNameConst.REAL_TIME_DATA_ANALYTICS_SUBSCRIBER_ID_RELATE_IP.equals(dslProfile.getName())) { - return dslService.realTimeDataAnalyticsSubscriberIdRelateIp(dslProfile); - } else if (DslIdentifierNameConst.REAL_TIME_DATA_ANALYTICS_MOBILE_IDENTITY_RELATE_TEID.equals(dslProfile.getName())) { - return dslService.realTimeDataAnalyticsMobileIdentityRelateTeid(dslProfile); - } - throw new BusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), dslProfile.getName())); - } -}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/qgw/controller/HosController.java b/src/main/java/com/mesalab/qgw/controller/HosController.java new file mode 100644 index 00000000..34f5c302 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/controller/HosController.java @@ -0,0 +1,84 @@ +package com.mesalab.qgw.controller; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.AuditLog; +import com.mesalab.qgw.service.HosService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.HttpServletRequest; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.stream.Collectors; + +@RestController +@RequestMapping(value = "/v1/hos") +public class HosController { + private static final Log log = LogFactory.get(); + private static final String X_HOS_META_PREFIX = "x-hos-meta-"; + + private HosService hosService; + + + @GetMapping(value = "/{bucket_name}/{file_name}", consumes = "application/x-www-form-urlencoded") + @AuditLog("HosController.getFile") + public BaseResult getFile(@PathVariable("bucket_name") String bucketName, @PathVariable("file_name") String fileName) { + return hosService.getFile(bucketName, fileName); + } + + @GetMapping(value = "/{bucket_name}", consumes = "application/x-www-form-urlencoded") + @AuditLog("HosController.getFileList") + public BaseResult getFileList(@PathVariable("bucket_name") String bucketName, @RequestParam(required = false) String prefix, HttpServletRequest request) { + if (StrUtil.isNotBlank(bucketName)) { + Map<String, String> udfMetaParam = new HashMap<>(); + Enumeration<String> parameterNames = request.getParameterNames(); + while (parameterNames.hasMoreElements()) { + String paramName = parameterNames.nextElement(); + if (paramName.startsWith(X_HOS_META_PREFIX)) { + String paramValue = request.getParameter(paramName); + udfMetaParam.put(paramName, paramValue); + } + } + log.info("Get File List interface,param:bucket_name is {},prefix is {},x-hos-meta-* is {}", bucketName, prefix, udfMetaParam.toString()); + return hosService.getFileList(bucketName, prefix, udfMetaParam); + } else { + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.BUCKET_NAME_ERROR)); + } + } + + @PostMapping(value = "/{bucket_name}/{file_name}", consumes = "multipart/form-data") + @AuditLog("HosController.uploadFile") + public BaseResult uploadFile(@PathVariable("bucket_name") String bucketName, @PathVariable("file_name") String fileName, @RequestParam("file") MultipartFile file) { + ByteArrayInputStream byteArrayInputStream = null; + try { + byteArrayInputStream = new ByteArrayInputStream(file.getBytes()); + } catch (IOException e) { + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), e.getMessage())); + } + return hosService.uploadFile(bucketName, fileName, byteArrayInputStream, null); + } + + @DeleteMapping(value = "/{bucket_name}", consumes = "application/x-www-form-urlencoded") + @AuditLog("HosController.deleteFileList") + public BaseResult deleteFileList(@PathVariable("bucket_name") String bucketName, @RequestParam("file_names") String fileNames) { + return hosService.deleteFileList(bucketName, Arrays.stream(fileNames.split(",")).distinct().collect(Collectors.toList())); + } + + + @Autowired + public void setHosService(HosService hosService) { + this.hosService = hosService; + } +} diff --git a/src/main/java/com/mesalab/qgw/controller/MetadataController.java b/src/main/java/com/mesalab/qgw/controller/MetadataController.java deleted file mode 100644 index e1cc8c10..00000000 --- a/src/main/java/com/mesalab/qgw/controller/MetadataController.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.mesalab.qgw.controller; - -import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.util.NumberUtil; -import com.alibaba.fastjson2.JSON; -import com.jayway.jsonpath.JsonPath; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.MetadataTypeEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.service.MetadataService; -import com.geedgenetworks.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@Slf4j -@RestController -@RequestMapping(value = "v1/metadata") -public class MetadataController { - - @Autowired - private MetadataService metadataService; - - private final static List<String> VISIBILITY = Arrays.asList("enabled", "hidden", "disabled"); - - - @RequestMapping("/schema/{table_name}") - public BaseResult getSchemaByTable(@PathVariable("table_name") String tableName) { - log.debug("SCHEMA信息获取,参数为{} {}", tableName); - - if (StringUtil.isBlank(tableName)) { - return BaseResultGenerator.success4Message("ok"); - } - return BaseResultGenerator.success(metadataService.getSchemaInfo(MetadataTypeEnum.FIELDS.getValue(), tableName, true)); - } - - @PutMapping("/schema/{table_name}") - public BaseResult updateSchema(@PathVariable("table_name") String tableName, @RequestBody Map<String, Object> obj) { - log.info("update Schema api, params is: {}-{}", tableName, obj); - if (StringUtil.isBlank(tableName) || StringUtil.isEmpty(obj)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_PARAM_ERROR)); - } - validationParam(tableName, obj); - return metadataService.updateSchema(tableName, obj); - } - - @RequestMapping("/{database}/table") - public BaseResult getTableByDB(@PathVariable String database) { - log.debug("SCHEMA信息获取,参数为{} {}",database); - - if (StringUtil.isBlank(database)) { - - return BaseResultGenerator.success4Message("ok"); - } - return BaseResultGenerator.success(metadataService.getSchemaInfo(MetadataTypeEnum.TABLES.getValue(), database, true)); - } - - private void validationParam(String name, Map<String, Object> obj) { - if (!metadataService.getAllTable().contains(name)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), name + QGWMessageConst.NOT_SUPPORT_SETTING_TTL)); - } - String param = JSON.toJSONString(obj); - if (!VISIBILITY.containsAll(JsonPath.read(param, "$.fields[?(@.doc.visibility != null)].doc.visibility"))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_ERROR_VISIBILITY_VALUE)); - } - List<Object> schemaDocTTL = JsonPath.read(param, "$.[?(@.doc.ttl != null)].doc.ttl"); - if (StringUtil.isNotEmpty(schemaDocTTL.stream().filter(o -> !(NumberUtil.isLong(String.valueOf(o)))).toArray())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_ERROR_TTL_VALUE)); - } - List<Object> fieldDocTTL = JsonPath.read(param, "$.fields[?(@.doc.ttl != null)].doc.ttl"); - Object[] objects3 = fieldDocTTL.parallelStream().filter(o -> !NumberUtil.isLong(String.valueOf(o))).toArray(); - if (StringUtil.isNotEmpty(objects3)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_ERROR_TTL_VALUE)); - } - if (schemaDocTTL.size() > 0 && fieldDocTTL.size() > 0 - && CollectionUtil.max(schemaDocTTL.stream().map(o -> Long.parseLong(o.toString())).collect(Collectors.toList())) < CollectionUtil.max(fieldDocTTL.stream().map(o -> Long.parseLong(o.toString())).collect(Collectors.toList()))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_ERROR_TTL_SIZE)); - } - } -} diff --git a/src/main/java/com/mesalab/qgw/controller/QueryController.java b/src/main/java/com/mesalab/qgw/controller/QueryController.java index 689ee8b9..be534094 100644 --- a/src/main/java/com/mesalab/qgw/controller/QueryController.java +++ b/src/main/java/com/mesalab/qgw/controller/QueryController.java @@ -1,44 +1,266 @@ package com.mesalab.qgw.controller; +import cn.hutool.core.util.BooleanUtil; import com.alibaba.fastjson2.JSON; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.model.basic.AuditLog; -import com.mesalab.qgw.model.basic.CachedSubmitCheck; -import com.mesalab.qgw.service.QueryService; +import com.mesalab.common.enums.ExecutionMode; +import com.mesalab.common.enums.OutputMode; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.*; +import com.mesalab.qgw.service.DeferredResultHolder; +import com.mesalab.qgw.service.QueryJobService; +import com.mesalab.qgw.service.SQLSyncQueryService; +import com.mesalab.services.configuration.JobConfig; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.context.request.async.DeferredResult; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; @Slf4j @RestController -@RequestMapping(value = "/sql") public class QueryController { - @Autowired - private QueryService queryService; + private SQLSyncQueryService sqlSyncQueryService; + private QueryJobService queryJobService; + private JobConfig jobConfig; + private DeferredResultHolder deferredResultHolder; - @GetMapping + @GetMapping("/sql") @AuditLog("QueryController.get") - @CachedSubmitCheck - public BaseResult get(QueryProfile queryProfile) { - if (queryProfile.queryParamIsNull()) + public BaseResult get(SQLQueryContext queryContext) { + if (queryContext.originalSQLIsNull()) return BaseResultGenerator.success4Message("ok"); - log.debug("Quest Params:{}", JSON.toJSONString(queryProfile)); - return queryService.executeQuery(queryProfile); + log.debug("Quest Params:{}", JSON.toJSONString(queryContext)); + return sqlSyncQueryService.executeQuery(queryContext); } - @PostMapping(produces = "application/json;charset=utf-8") + @PostMapping(value = "/sql", produces = "application/json;charset=utf-8") @AuditLog("QueryController.post") - @CachedSubmitCheck - public BaseResult post(@RequestBody QueryProfile queryProfile) { - if (queryProfile.queryParamIsNull()) { + public BaseResult post(@RequestBody SQLQueryContext queryContext) { + if (queryContext.originalSQLIsNull()) { return BaseResultGenerator.success4Message("ok"); } - log.debug("Quest Params:{}", JSON.toJSONString(queryProfile)); - return queryService.executeQuery(queryProfile); + log.debug("Quest Params:{}", JSON.toJSONString(queryContext)); + return sqlSyncQueryService.executeQuery(queryContext); + + } + + @PostMapping(value = "/v1/query/sql", consumes = "application/json", produces = "application/json") + @AuditLog("QueryController.createSQLQuery") + public BaseResult createSQLQuery(@RequestBody SqlQueryRequestParam sqlQueryRequestParam) { + log.info("Create an SQL Query {}", JSON.toJSONString(sqlQueryRequestParam)); + if (sqlQueryRequestParam.isSavedQuery()) { + if (sqlQueryRequestParam.isDryRun() || !sqlQueryRequestParam.getExecutionMode().equals(ExecutionMode.NORMAL) || !sqlQueryRequestParam.getOutputMode().equals(OutputMode.JSON)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getMessage(), + "for the Saved Query Job, is_dry_run must be 0(default), execution_mode must be normal(default), and output_mode must be JSON(default)")); + } + return queryJobService.createSQLSavedQuery(sqlQueryRequestParam); + } else { + return queryJobService.createSQLAdHocQuery(sqlQueryRequestParam); + } + } + + @PostMapping(value = "/v1/query/dsl", consumes = "application/json", produces = "application/json") + @AuditLog("QueryController.createDSLQuery") + public BaseResult createDSLQuery(@RequestBody DSLQueryRequestParam dslQueryRequestParam) { + log.info("Create a DSL query {}", JSON.toJSONString(dslQueryRequestParam)); + if (dslQueryRequestParam.isSavedQuery()) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getMessage(), "Currently, the DSL Saved Query is not supported.")); + } else { + if (!dslQueryRequestParam.getOutputMode().equals(OutputMode.JSON)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getMessage(), "Currently, the DSL Query not support output_mode must be JSON(default)")); + } + return queryJobService.createDSLAdHocQuery(dslQueryRequestParam); + } + } + + @GetMapping(value = "/v1/query/job/{id}/result", consumes = "application/x-www-form-urlencoded") + @AuditLog("QueryController.getJobResultById") + public DeferredResult<BaseResult> getJobResultById(@PathVariable String id, @RequestParam(value = "is_saved_query", required = false, defaultValue = "0") Integer isSavedQuery) { + BaseResult baseResult; + if (BooleanUtil.toBoolean(String.valueOf(isSavedQuery))) { + baseResult = queryJobService.getSavedQueryResultById(id); + } else { + baseResult = queryJobService.getAdHocQueryResultById(id); + } + DeferredResult<BaseResult> deferredResult = + deferredResultHolder.newDeferredResult(id, jobConfig.getResponseTimeout(), baseResult); + if (!jobConfig.isLongPollingEnabled()) { + deferredResultHolder.handleDeferredData(deferredResult, baseResult); + return deferredResult; + } + deferredResult.onTimeout(() -> { + deferredResultHolder.handleDeferredData(deferredResult, BooleanUtil.toBoolean(String.valueOf(isSavedQuery)) ? queryJobService.getSavedQueryResultById(id) : queryJobService.getAdHocQueryResultById(id)); + }); + + if (StringUtil.isNotEmpty(baseResult) + && StringUtil.isNotEmpty(baseResult.getJob()) + && Boolean.valueOf(String.valueOf(baseResult.getJob().get(JobConfig.IS_DONE)))) { + deferredResultHolder.handleDeferredData(deferredResult, baseResult); + } + return deferredResult; + } + + @GetMapping(value = "/v1/query/job/{id}", consumes = "application/x-www-form-urlencoded") + @AuditLog("QueryController.getJobStatusById") + public DeferredResult<BaseResult> getJobStatusById(@PathVariable String id, @RequestParam(value = "is_saved_query", required = false, defaultValue = "0") Integer isSavedQuery) { + BaseResult baseResult; + if (BooleanUtil.toBoolean(String.valueOf(isSavedQuery))) { + baseResult = queryJobService.getSavedQueryStatusById(id); + } else { + baseResult = queryJobService.getAdHocQueryStatusById(id); + } + DeferredResult<BaseResult> deferredResult = + deferredResultHolder.newDeferredResult(id, jobConfig.getResponseTimeout(), baseResult); + if (!jobConfig.isLongPollingEnabled()) { + deferredResultHolder.handleDeferredData(deferredResult, baseResult); + return deferredResult; + } + deferredResult.onTimeout(() -> { + deferredResultHolder.handleDeferredData(deferredResult, BooleanUtil.toBoolean(String.valueOf(isSavedQuery)) ? queryJobService.getSavedQueryStatusById(id) : queryJobService.getAdHocQueryStatusById(id)); + }); + if (StringUtil.isNotEmpty(baseResult) + && StringUtil.isNotEmpty(baseResult.getJob()) + && Boolean.valueOf(String.valueOf(baseResult.getJob().get(JobConfig.IS_DONE)))) { + deferredResultHolder.handleDeferredData(deferredResult, baseResult); + } + return deferredResult; + } + + @GetMapping(value = "/v1/query/job/result", consumes = "application/x-www-form-urlencoded") + @AuditLog("QueryController.getJobResult") + public DeferredResult<BaseResult> getJobResult(@RequestParam(value = "job_ids") String jobIds + , @RequestParam(value = "is_saved_query", required = false, defaultValue = "0") Integer isSavedQueryInt) { + boolean isSavedQuery = BooleanUtil.toBoolean(String.valueOf(isSavedQueryInt)); + List<String> ids = Arrays.stream(jobIds.split(",")).distinct().collect(Collectors.toList()); + BaseResult baseResult; + if (isSavedQuery) { + baseResult = queryJobService.getSavedQueryResult(ids); + } else { + baseResult = queryJobService.getAdHocQueryResult(ids); + } + + DeferredResult<BaseResult> deferredResult = + deferredResultHolder.newDeferredResult(String.valueOf(ids), jobConfig.getResponseTimeout(), baseResult); + if (!jobConfig.isLongPollingEnabled()) { + deferredResultHolder.handleDeferredData(deferredResult, baseResult); + return deferredResult; + } + deferredResult.onTimeout(() -> { + deferredResultHolder.handleDeferredData(deferredResult, BooleanUtil.toBoolean(String.valueOf(isSavedQuery)) ? queryJobService.getSavedQueryResult(ids) : queryJobService.getAdHocQueryResult(ids)); + }); + + if (jobsIsDoneOfResult(baseResult)) { + deferredResultHolder.handleDeferredData(deferredResult, baseResult); + } + return deferredResult; + } + + @GetMapping(value = "/v1/query/job", consumes = "application/x-www-form-urlencoded") + @AuditLog("QueryController.getJobStatus") + public DeferredResult<BaseResult> getJobStatus(@RequestParam(value = "job_ids") String jobIds + , @RequestParam(value = "is_saved_query", required = false, defaultValue = "0") Integer isSavedQueryInt) { + boolean isSavedQuery = BooleanUtil.toBoolean(String.valueOf(isSavedQueryInt)); + List<String> ids = Arrays.stream(jobIds.split(",")).distinct().collect(Collectors.toList()); + BaseResult baseResult; + if (isSavedQuery) { + baseResult = queryJobService.getSavedQueryStatus(ids); + } else { + baseResult = queryJobService.getAdHocQueryStatus(ids); + } + + DeferredResult<BaseResult> deferredResult = + deferredResultHolder.newDeferredResult(String.valueOf(ids), jobConfig.getResponseTimeout(), baseResult); + if (!jobConfig.isLongPollingEnabled()) { + deferredResultHolder.handleDeferredData(deferredResult, baseResult); + return deferredResult; + } + deferredResult.onTimeout(() -> { + deferredResultHolder.handleDeferredData(deferredResult, BooleanUtil.toBoolean(String.valueOf(isSavedQuery)) ? queryJobService.getSavedQueryStatus(ids) : queryJobService.getAdHocQueryStatus(ids)); + }); + + if (jobsIsDoneOfStatus(baseResult)) { + deferredResultHolder.handleDeferredData(deferredResult, baseResult); + } + return deferredResult; + } + + @DeleteMapping(value = "/v1/query/job/{id}", consumes = "application/x-www-form-urlencoded") + @AuditLog("QueryController.deleteJobById") + public BaseResult deleteJobById(@PathVariable String id, @RequestParam(value = "is_saved_query", required = false, defaultValue = "0") Integer isSavedQueryInt) { + boolean isSavedQuery = BooleanUtil.toBoolean(String.valueOf(isSavedQueryInt)); + if (isSavedQuery) { + return queryJobService.deleteSavedQueryById(id); + } + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getMessage(), "The AdHoc Query Job does not yet support deletion.")); + } + + private boolean jobsIsDoneOfResult(BaseResult baseResult) { + if (StringUtil.isEmpty(baseResult)) { + return false; + } + if (StringUtil.isEmpty(baseResult.getData())) { + return false; + } + List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); + for (Map<String, Object> datum : data) { + Map<String, Object> job = (Map<String, Object>) datum.get(JobConfig.JOB); + if (!BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_DONE)))) { + return false; + } + } + return true; + } + + private boolean jobsIsDoneOfStatus(BaseResult baseResult) { + if (StringUtil.isEmpty(baseResult)) { + return false; + } + if (StringUtil.isEmpty(baseResult.getData())) { + return false; + } + List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); + for (Map<String, Object> datum : data) { + if (!BooleanUtil.toBoolean(String.valueOf(datum.get(JobConfig.IS_DONE)))) { + return false; + } + } + return true; + } + + @Autowired + public void setQueryJobService(QueryJobService queryJobService) { + this.queryJobService = queryJobService; + } + + @Autowired + public void setSqlSyncQueryService(SQLSyncQueryService sqlSyncQueryService) { + this.sqlSyncQueryService = sqlSyncQueryService; + } + + @Autowired + public void setJobConfig(JobConfig jobConfig) { + this.jobConfig = jobConfig; + } + + @Autowired + public void setDeferredResultHolder(DeferredResultHolder deferredResultHolder) { + this.deferredResultHolder = deferredResultHolder; } } diff --git a/src/main/java/com/mesalab/qgw/controller/SystemController.java b/src/main/java/com/mesalab/qgw/controller/SystemController.java index 23fa01e2..db8d78fa 100644 --- a/src/main/java/com/mesalab/qgw/controller/SystemController.java +++ b/src/main/java/com/mesalab/qgw/controller/SystemController.java @@ -2,93 +2,19 @@ package com.mesalab.qgw.controller; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.clearspring.analytics.util.Lists; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.qgw.model.job.EncryptionInfo; -import com.mesalab.qgw.model.job.StorageDeletionInfo; -import com.mesalab.qgw.service.SystemService; import com.mesalab.services.configuration.ThreadPoolMonitor; -import com.geedgenetworks.utils.StringUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; -import java.util.List; -import java.util.Map; - @RestController @RequestMapping(value = "/v1/admin") public class SystemController { private static final Log log = LogFactory.get(); @Autowired - private SystemService systemService; - @Autowired ThreadPoolMonitor threadPoolMonitor; - - @GetMapping ("setting/storage/quota") - public BaseResult storageQuota() { - return systemService.getStorageQuota(); - } - - @PutMapping(value = "setting/storage/quota") - public BaseResult storageSetting(@RequestBody List<StorageDeletionInfo> list) { - log.warn("数据配额设置, 参数: params is {}", list); - if (StringUtil.isEmpty(list)) { - return BaseResultGenerator.failure(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.PARAMETER_NOT_OBTAINED)); - } - for (StorageDeletionInfo info : list) { - if (StringUtil.isBlank(info.getType()) || StringUtil.isBlank(String.valueOf(info.getMaxDays()))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.LOGTYPE_AND_MAXDAYS_NOT_NULL)); - } - if ("ALL".equalsIgnoreCase(info.getType()) && list.size() != 1) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.CANNOT_COEXIST_OTHER_TYPES)); - } - if (!(info.getMaxDays() >= 0 && info.getMaxDays() <= 2000)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.MAXDAYS_ERROR)); - } - } - return systemService.deleteStorage(list); - } - - @RequestMapping(value = "setting/storage/quota",method = RequestMethod.DELETE) - public BaseResult storageDeletion(@RequestParam String type){ - List<StorageDeletionInfo> deletionInfoList = Lists.newArrayList(); - StorageDeletionInfo deletionInfo = new StorageDeletionInfo(); - deletionInfo.setType(type); - deletionInfo.setMaxDays(0); - deletionInfoList.add(deletionInfo); - return systemService.deleteStorage(deletionInfoList); - } - - @RequestMapping("setting/storage/quota/daily_trend") - public BaseResult dailyTrendOfStorage(@RequestParam Map<String, Object> param) { - String searchStartTime = StringUtil.stripToEmpty((String) param.get("start_time")); - String searchEndTime = StringUtil.stripToEmpty((String) param.get("end_time")); - return systemService.dailyTrendOfStorage(searchStartTime, searchEndTime); - } - - - @PostMapping(value = "setting/tool/password_encryption") - public BaseResult getCiphertext(EncryptionInfo param) { - log.info("Plaintext encrypted,The plaintext argument is : {}", param); - if (StringUtil.isEmpty(param.getPassword()) || StringUtil.isEmpty(param.getSalt())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.PASSWORD_OR_SALT_CANNOT_BE_EMPTY)); - } - return systemService.getCiphertext(param); - } - @GetMapping(value = "/clear/realJobQueue") public BaseResult clearTaskQueue() { log.info("Clear real job queue."); diff --git a/src/main/java/com/mesalab/qgw/controller/TestController.java b/src/main/java/com/mesalab/qgw/controller/TestController.java index fc2e84f2..4ce602c5 100644 --- a/src/main/java/com/mesalab/qgw/controller/TestController.java +++ b/src/main/java/com/mesalab/qgw/controller/TestController.java @@ -29,7 +29,7 @@ import java.util.List; public class TestController { @Autowired - private DiagnosisService diagnosisService; + TroubleshootingController troubleshootingController; @Autowired private ProjectInfo projectProperties; @Autowired @@ -56,6 +56,7 @@ public class TestController { * @return BaseResult */ @GetMapping(value = "projectProperties") + @AuditLog("TestController.projectProperties") public BaseResult projectProperties() { log.info("Project Properties: {}", projectProperties); return BaseResultGenerator.success("ok", projectProperties); @@ -67,6 +68,7 @@ public class TestController { * @return 返回当前项目的环境值 */ @GetMapping(value = "activeProfiles") + @AuditLog("TestController.activeProfiles") public BaseResult activeProfiles() { List<String> activeProfiles = Lists.newArrayList(nacosConfig.getNamespace()); log.info("Current Namespace: {}", activeProfiles); @@ -79,6 +81,7 @@ public class TestController { * @return 返回当前项目的运行环境 */ @GetMapping(value = "env") + @AuditLog("TestController.env") public BaseResult env() { return BaseResultGenerator.success(Lists.newArrayList(nacosConfig.getNamespace())); } @@ -89,41 +92,18 @@ public class TestController { * @return 检查授权 */ @GetMapping(value = "checkAuthc") + @AuditLog("TestController.checkAuthc") public BaseResult checkAuthc() { return BaseResultGenerator.success(); } - - /** - * 运行测试用例 - * - * @return - */ - @GetMapping(value = "runSql") - @Deprecated - public BaseResult runSql() { - BaseResult result = diagnosisService.runPocSQL(false, "validation", null); - return result; - } - - /** - * 校验schema - * - * @return - */ - @GetMapping(value = "runSchema") - @Deprecated - public BaseResult runSchema() { - BaseResult result = diagnosisService.validateSchema(); - return result; - } - /** * 检查job动态连接配置 * * @return */ @GetMapping(value = "jobAdminHttpSource") + @AuditLog("TestController.jobAdminHttpSource") public BaseResult jobAdminHttpSource() { return BaseResultGenerator.success("ok", jobAdminHttpSource); } @@ -134,6 +114,7 @@ public class TestController { * @return */ @GetMapping(value = "druidIoHttpSource") + @AuditLog("TestController.druidIoHttpSource") public BaseResult druidIoHttpSource() { return BaseResultGenerator.success("ok", druidIoHttpSource); } @@ -144,6 +125,7 @@ public class TestController { * @return */ @GetMapping(value = "clickHouseHttpSource") + @AuditLog("TestController.clickHouseHttpSource") public BaseResult clickHouseHttpSource() { return BaseResultGenerator.success("ok", clickHouseHttpSource); } @@ -154,6 +136,7 @@ public class TestController { * @return */ @GetMapping(value = "hBaseAPISource") + @AuditLog("TestController.hBaseAPISource") public BaseResult hBaseAPISource() { return BaseResultGenerator.success("ok", hBaseAPISource); } @@ -164,6 +147,7 @@ public class TestController { * @return */ @GetMapping(value = "engineConfigSource") + @AuditLog("TestController.engineConfigSource") public BaseResult engineConfigSource() { return BaseResultGenerator.success("ok", engineConfigSource); } @@ -174,6 +158,7 @@ public class TestController { * @return */ @GetMapping(value = "arangoConfig") + @AuditLog("TestController.arangoConfig") public BaseResult arangoConfig() { return BaseResultGenerator.success("ok", arangoConfig); } @@ -184,6 +169,7 @@ public class TestController { * @return */ @GetMapping(value = "httpConfig") + @AuditLog("TestController.httpConfig") public BaseResult httpConfig() { return BaseResultGenerator.success("ok", httpConfig); } diff --git a/src/main/java/com/mesalab/qgw/controller/TroubleshootingController.java b/src/main/java/com/mesalab/qgw/controller/TroubleshootingController.java new file mode 100644 index 00000000..0489d4d5 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/controller/TroubleshootingController.java @@ -0,0 +1,67 @@ +package com.mesalab.qgw.controller; + +import cn.hutool.core.util.BooleanUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.http.HttpStatus; +import com.alibaba.fastjson2.JSONObject; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.enums.*; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.AuditLog; +import com.mesalab.qgw.service.TroubleshootingService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import java.util.Arrays; + +/** + * Galaxy日志自动诊断接口 + */ +@Slf4j +@RestController +@RequestMapping(value = "/v1/troubleshooting") +public class TroubleshootingController { + + @Autowired + private TroubleshootingService troubleshootingService; + @RequestMapping("/component/status") + @AuditLog("TroubleshootingController.componentStatus") + public JSONObject componentStatus() { + return troubleshootingService.getComponentStatus(); + } + + @RequestMapping(value = "sanity", method = RequestMethod.GET) + @AuditLog("TroubleshootingController.sanity") + public BaseResult sanity(String test) { + if (Arrays.stream(ConsistencyOption.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(test))) { + throw new QGWBusinessException(HttpStatus.HTTP_BAD_REQUEST, CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.CONSISTENCY_OPTION_ERROR)); + } + if (ConsistencyOption.TTL_CONSISTENCY.getValue().equalsIgnoreCase(test)) { + log.info("ttl consistency check interface."); + return troubleshootingService.consistencyCheck(); + } else if (ConsistencyOption.DATASET_VERIFICATION.getValue().equalsIgnoreCase(test)) { + log.info("dataset verification interface. "); + return troubleshootingService.datesetVerification(); + } else { + log.info("schema consistency check interface."); + return troubleshootingService.validateMetadata(); + } + } + + @RequestMapping(value = "/benchmark", method = RequestMethod.GET) + @AuditLog("TroubleshootingController.benchmark") + public BaseResult benchmark(@RequestParam String test, @RequestParam(value = "is_saved", required = false, defaultValue = "0") @JsonProperty("isSaved") Integer isSaved) throws BusinessException { + if (StrUtil.isBlankIfStr(test)) { + throw new QGWBusinessException(HttpStatus.HTTP_BAD_REQUEST, CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.BENCHMARK_TEST_PARAM_CANNOT_BE_EMPTY)); + } + return troubleshootingService.benchmarkTest(test, BooleanUtil.toBoolean(String.valueOf(isSaved))); + } + +} diff --git a/src/main/java/com/mesalab/qgw/controller/UtilController.java b/src/main/java/com/mesalab/qgw/controller/UtilController.java new file mode 100644 index 00000000..c47ecd4d --- /dev/null +++ b/src/main/java/com/mesalab/qgw/controller/UtilController.java @@ -0,0 +1,41 @@ +package com.mesalab.qgw.controller; + +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.geedgenetworks.utils.StringUtil; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.qgw.model.basic.AuditLog; +import com.mesalab.qgw.model.job.EncryptionInfo; +import com.mesalab.qgw.service.UtilService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +@RestController +@RequestMapping(value = "/v1/util") +public class UtilController { + + private static final Log log = LogFactory.get(); + @Autowired + private UtilService utilService; + + @GetMapping(value = "/sql_parser", consumes = "application/x-www-form-urlencoded") + @AuditLog("UtilController.getSQLSyntaxTree") + public BaseResult getSQLSyntaxTree(String sql){ + return utilService.getSQLSyntaxTree(sql); + } + + @PostMapping(value = "/password_generator", consumes = "application/json") + @AuditLog("UtilController.getCiphertext") + public BaseResult getCiphertext(@RequestBody EncryptionInfo param) { + log.info("Plaintext encrypted,The plaintext argument is : {}", param); + if (StringUtil.isEmpty(param.getPassword()) || StringUtil.isEmpty(param.getSalt())) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.PASSWORD_OR_SALT_CANNOT_BE_EMPTY)); + } + return utilService.getCiphertext(param); + } +} diff --git a/src/main/java/com/mesalab/qgw/dialect/AbstractDataSourceDialect.java b/src/main/java/com/mesalab/qgw/dialect/AbstractDataSourceDialect.java index 02b55393..6724f2ed 100644 --- a/src/main/java/com/mesalab/qgw/dialect/AbstractDataSourceDialect.java +++ b/src/main/java/com/mesalab/qgw/dialect/AbstractDataSourceDialect.java @@ -1,61 +1,86 @@ package com.mesalab.qgw.dialect; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; import com.alibaba.fastjson2.JSON; import com.google.common.collect.Maps; import com.mesalab.common.entity.BaseResult; import com.geedgenetworks.utils.Encodes; +import com.mesalab.qgw.model.basic.SQLQueryContext; +import com.mesalab.qgw.model.basic.udf.TIME_FLOOR_WITH_FILL; +import com.mesalab.qgw.model.basic.udf.UDF; import lombok.Data; -import java.util.Map; -import java.util.Optional; +import java.util.*; @Data public abstract class AbstractDataSourceDialect implements Dialect { - abstract void init(); + protected SQLQueryContext sqlQueryContext; + private static final Log log = LogFactory.get(); + public AbstractDataSourceDialect(SQLQueryContext sqlQueryContext) { + this.sqlQueryContext = sqlQueryContext; + } - /** - * SQL查询语句转换为方言所认知的查询语句。 - * @param sql - * @return - */ - abstract String convertQuery(String sql); + abstract void init(); /** - * 基于RBO(rule-base optimizer)获取最优化的SQL + * Query SQL statement to be converted the SQL that can be executed by the data source. + * + * @param sql * @return */ - abstract String getBestQuery(); + abstract String convertQuery(String sql); /** - * 获取原始SQL语句，不做规则转换，直接下推 + * Generate the best query statement by RBO(Rule Based Optimization) + * * @return */ - abstract String getOriginalQuery(); + abstract String generateRuleBaseOptimizedQuery(); /** - * 获取样例SQL，目标生成无Filter语句。 + * 获取样例SQL，目标生成空执行语句。 + * * @return */ - abstract String getSampleQuery(); + abstract String getSampleQuery(); /** * get Explain Statement + * * @return */ abstract String getExplainQuery(); /** - * 简化规范异常信息展示。 - * @return + * Desc: 简化规范异常信息展示。 + * + * @return {@link String} */ abstract String simplyErrorMessage(Object object); + /** + * Desc: 构建样例数据 meta 信息 + * + * @created by wWei + * @date 2024/3/7 09:54 + */ + abstract List<Map<String, String>> buildExampleMeta(); + + /** + * Desc: 样例数据 + * + * @return {@link null} + * @created by wWei + * @date 2024/3/7 10:15 + */ + abstract List<Object> buildExampleData(); @Override final public BaseResult executeQuery() { init(); - String transformedSQL = convertQuery(getBestQuery()); + String transformedSQL = convertQuery(generateRuleBaseOptimizedQuery()); return generateBaseResult(transformedSQL, Optional.of("OK")); } @@ -63,45 +88,51 @@ public abstract class AbstractDataSourceDialect implements Dialect { final public BaseResult executeSampleQuery() { init(); String transformedSQL = getSampleQuery(); - return generateBaseResult(transformedSQL, Optional.of("OK")); } /** * 解析并生成最终结果 + * * @param sql * @param message * @return */ - abstract BaseResult generateBaseResult(String sql, Optional<String> message); + abstract BaseResult generateBaseResult(String sql, Optional<String> message); /** * shows the execution plan of a select statement. + * * @return */ final public BaseResult executeExplainPlan() { init(); String transformedSQL = getExplainQuery(); Map sqlMap = Maps.newHashMap(); - sqlMap.put("OriginalSQL", Encodes.encodeBase64(getOriginalQuery().getBytes())); - sqlMap.put("transformedSQL", Encodes.encodeBase64(transformedSQL.getBytes())); + sqlMap.put("OriginalSQL", Encodes.encodeBase64(sqlQueryContext.getOriginalSQL().getBytes())); + sqlMap.put("TransformedSQL", Encodes.encodeBase64(transformedSQL.getBytes())); return generateBaseResult(transformedSQL, Optional.of(JSON.toJSONString(sqlMap))); } - @Override - final public BaseResult executeSyntaxCheck() { + final public BaseResult executeSyntaxValidation() { init(); String transformedSQL = getSampleQuery(); - return generateBaseResult(transformedSQL, Optional.of(getOriginalQuery())); + BaseResult baseResult = generateBaseResult(transformedSQL, Optional.of(sqlQueryContext.getOriginalSQL())); + try { + baseResult.setMeta(buildExampleMeta()); + baseResult.setData(buildExampleData()); + } catch (Exception e) { + log.error("Build Example Data Error: {} ", e.toString()); + } + return baseResult; } - @Override final public BaseResult executeAdministrativeQuery() { init(); - String sql = getOriginalQuery(); - return generateBaseResult(sql, Optional.of("OK")); + return generateBaseResult(sqlQueryContext.getOriginalSQL(), Optional.of("OK")); } + } diff --git a/src/main/java/com/mesalab/qgw/dialect/AbstractEngineDialect.java b/src/main/java/com/mesalab/qgw/dialect/AbstractEngineDialect.java index be1a3e01..e52248db 100644 --- a/src/main/java/com/mesalab/qgw/dialect/AbstractEngineDialect.java +++ b/src/main/java/com/mesalab/qgw/dialect/AbstractEngineDialect.java @@ -15,12 +15,12 @@ public abstract class AbstractEngineDialect implements Dialect { /** * Desc: 执行自定义函数操作 * - * @param + * @param isDryRun * @return * @created by wWei * @date 2022/4/27 5:25 下午 */ - abstract void executeUDF(); + abstract void executeUDF(boolean isDryRun); /** * Desc: 获取联合查询SQL @@ -62,14 +62,14 @@ public abstract class AbstractEngineDialect implements Dialect { @Override final public BaseResult executeQuery() { init(); - executeUDF(); + executeUDF(false); return generateBaseResult(getFederateQuery()); } @Override final public BaseResult executeSampleQuery() { init(); - executeUDF(); + executeUDF(false); return generateBaseResult(getSampleQuery()); } @@ -79,10 +79,10 @@ public abstract class AbstractEngineDialect implements Dialect { } @Override - final public BaseResult executeSyntaxCheck() { + final public BaseResult executeSyntaxValidation() { init(); - executeUDF(); - return generateBaseResult(getSyntaxCheckQuery()); + executeUDF(true); + return generateBaseResult(getSampleQuery()); } @Override diff --git a/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java b/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java index 2597f3f7..4873b559 100644 --- a/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java +++ b/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java @@ -1,13 +1,13 @@ package com.mesalab.qgw.dialect; -import cn.hutool.core.util.BooleanUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.URLUtil; import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONPath; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.jayway.jsonpath.JsonPath; @@ -17,26 +17,29 @@ import com.mesalab.common.entity.DataTypeMapping; import com.mesalab.common.enums.*; import com.mesalab.common.utils.*; import com.mesalab.common.utils.sqlparser.*; +import com.mesalab.qgw.constant.MetaConst; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.udf.ROLLUP; import com.mesalab.qgw.service.RewriteTable; import com.mesalab.qgw.model.basic.HttpConfig; import com.mesalab.qgw.model.basic.*; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.qgw.service.impl.HttpClientService; +import com.mesalab.qgw.service.DatabaseService; import com.geedgenetworks.utils.Encodes; import com.geedgenetworks.utils.StringUtil; +import com.mesalab.qgw.service.impl.HttpClientServiceV2; +import lombok.SneakyThrows; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.select.*; import net.sf.jsqlparser.util.deparser.ExpressionDeParser; -import org.apache.avro.Schema; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; import org.springframework.util.CollectionUtils; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -44,6 +47,7 @@ import java.util.regex.Pattern; public class ClickHouseDialect extends AbstractDataSourceDialect { private static final Log log = LogFactory.get(); private static final Pattern pNullable = Pattern.compile("\\bNullable\\((.*?)\\)", Pattern.CASE_INSENSITIVE); + private static final Pattern pDateTime = Pattern.compile("\\b(DateTime|DateTime64|Date)\\((.*?)\\)", Pattern.CASE_INSENSITIVE); private static final Pattern pIn = Pattern.compile("(?<!\\bnot)\\s+in\\s*\\(\\s*SELECT\\b", Pattern.CASE_INSENSITIVE); private static final Pattern pNotIn = Pattern.compile("\\bnot\\s+in\\s*\\(\\s*SELECT\\b", Pattern.CASE_INSENSITIVE); private static final Pattern pUniq = Pattern.compile("count\\s*\\(\\s*distinct\\s*\\((.*?)\\)", Pattern.CASE_INSENSITIVE); @@ -51,16 +55,14 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { private static final Pattern pWhere = Pattern.compile("where\\s*", Pattern.CASE_INSENSITIVE); private static final Pattern pGroupBy = Pattern.compile("\\bgroup\\s+by\\s+", Pattern.CASE_INSENSITIVE); private static final Pattern pDistinct = Pattern.compile("\\b(DISTINCT)(\\(|\\s+)", Pattern.CASE_INSENSITIVE); - private static final Pattern pAggregateFunSpecificOfCK = Pattern.compile("\\b(anyHeavy|anyLast|argMin|argMax|avgWeighted|topK|topKWeighted|groupArray|groupUniqArray|groupArrayInsertAt|groupArrayMovingAvg|groupArrayMovingSum|groupBitAnd|groupBitOr|groupBitXor|groupBitmap|groupBitmapAnd|groupBitmapOr|groupBitmapXor|sumWithOverflow|sumMap|minMap|maxMap|skewSamp|skewPop|kurtSamp|kurtPop|uniq|uniqExact|uniqCombined|uniqCombined64|uniqHLL12|quantile|quantiles|quantileExact|quantileExactLow|quantileExactHigh|quantileExactWeighted|quantileTiming|quantileTimingWeighted|quantileDeterministic|quantileTDigest|quantileTDigestWeighted|quantileBFloat16|quantileBFloat16Weighted|simpleLinearRegression|stochasticLinearRegression|stochasticLogisticRegression|categoricalInformationValue)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static final Pattern pAggregateFunSpecificOfCK = Pattern.compile("\\b(anyHeavy|anyLast|argMin|argMax|avgWeighted|topK|topKWeighted|groupArray|groupUniqArray|groupArrayInsertAt|groupArrayMovingAvg|groupArrayMovingSum|groupBitAnd|groupBitOr|groupBitXor|groupBitmap|groupBitmapAnd|groupBitmapOr|groupBitmapXor|sumWithOverflow|sumMap|minMap|maxMap|skewSamp|skewPop|kurtSamp|kurtPop|uniq|uniqExact|uniqCombined|uniqCombined64|uniqHLL12|quantile|quantiles|quantileExact|quantileExactLow|quantileExactHigh|quantileExactWeighted|quantileTiming|quantileTimingWeighted|quantileDeterministic|quantileTDigest|quantileTDigestWeighted|quantileBFloat16|quantileBFloat16Weighted|simpleLinearRegression|stochasticLinearRegression|stochasticLogisticRegression|categoricalInformationValue)\\s*\\(", Pattern.CASE_INSENSITIVE); private static final Pattern pCount1 = Pattern.compile("count\\(\\s*1\\s*\\)", Pattern.CASE_INSENSITIVE); - private ClickHouseHttpSource clickHouseHttpSource = (ClickHouseHttpSource) SpringContextUtil.getBean("clickHouseHttpSource"); - private HttpClientService httpClientService = (HttpClientService) SpringContextUtil.getBean("httpClientService"); - private MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("metadataService"); - private EngineConfigSource engineConfigSource = (EngineConfigSource) SpringContextUtil.getBean("engineConfigSource"); - private HttpConfig httpConfig = (HttpConfig) SpringContextUtil.getBean("httpConfig"); - public QueryProfile param; + private final ClickHouseHttpSource clickHouseHttpSource = (ClickHouseHttpSource) SpringContextUtil.getBean("clickHouseHttpSource"); + private final HttpClientServiceV2 httpClientService = (HttpClientServiceV2) SpringContextUtil.getBean("httpClientServiceV2"); + private final DatabaseService databaseService = (DatabaseService) SpringContextUtil.getBean("databaseService"); + private final EngineConfigSource engineConfigSource = (EngineConfigSource) SpringContextUtil.getBean("engineConfigSource"); + private final HttpConfig httpConfig = (HttpConfig) SpringContextUtil.getBean("httpConfig"); public BaseResult baseResult; - private final static double PERCENT_UP_LINE = 0.98; private final static double PERCENT_DOWN_LINE = 0.01; private final static String DB_EXCEPTION_START = "e.displayText() = DB::Exception: "; @@ -86,19 +88,19 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { metaMap.put("DateTime", DataTypeMapping.TIMESTAMP); } - public ClickHouseDialect(QueryProfile param) { - this.param = param; + public ClickHouseDialect(SQLQueryContext queryContext) { + super(queryContext); } @Override public void init() { - if (StringUtil.isNotEmpty(param.getDbQuerySource())) { - param.getDbQuerySource().setExecSQL(param.getDbQuerySource().getSqlBody()); + if (StringUtil.isNotEmpty(sqlQueryContext.getDbSelectStatement())) { + sqlQueryContext.getDbSelectStatement().setExecSQL(sqlQueryContext.getDbSelectStatement().getSqlBody()); } else { - SQLQuerySource sqlQuerySource = new SQLQuerySource(); - sqlQuerySource.setExecSQL(param.getQuery()); - param.setDbQuerySource(sqlQuerySource); + SelectStatement selectStatement = new SelectStatement(); + selectStatement.setExecSQL(sqlQueryContext.getOriginalSQL()); + sqlQueryContext.setDbSelectStatement(selectStatement); } baseResult = BaseResultGenerator.success(); } @@ -116,16 +118,11 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { } @Override - public String getBestQuery() { + public String generateRuleBaseOptimizedQuery() { return new Optimizer().generateQueryLimit(). generateSQL().generateSubQuery().generateTableAlias().generateApproximation().build(); } - @Override - public String getOriginalQuery() { - return param.getQuery(); - } - /** * 获取语法检测sql * @@ -133,50 +130,48 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { */ @Override public String getSampleQuery() { - String sampleSQL = param.getDbQuerySource().getSqlBody(); + String sampleSQL = sqlQueryContext.getDbSelectStatement().getSqlBody(); try { sampleSQL = convertQuery(convertSampleSql(sampleSQL)); } catch (JSQLParserException e) { log.error("Sample sql error {}, execute original sql: {}, error is: {} ", - sampleSQL, sampleSQL = param.getDbQuerySource().getSqlBody(), e.getMessage() == null ? e.getCause() : e.getMessage()); + sampleSQL, sampleSQL = sqlQueryContext.getDbSelectStatement().getSqlBody(), e.getMessage() == null ? e.getCause() : e.getMessage()); } return sampleSQL; } @Override String getExplainQuery() { - return "explain " + convertQuery(getBestQuery()); + return "explain " + convertQuery(generateRuleBaseOptimizedQuery()); } @Override public BaseResult executeKillQuery(String queryId) { - String killSql = StrUtil.format("kill query where query_id ='{}'", queryId); + String killSql = StrUtil.format("kill query on cluster ck_cluster where is_initial_query = 1 and query_id ='{}'", queryId); String queryURL = URLUtil.normalize(clickHouseHttpSource.getUrl() + "/?"); StringBuilder queryParamBuilder = new StringBuilder(getLongTermAccount()).append("&") - .append("database=").append(clickHouseHttpSource.getDbName()).append("&") - .append("query=").append(killSql) - .append(" FORMAT ").append(QueryFormatEnum.JSON.getValue().toUpperCase()).append(" ;"); - List<NameValuePair> values = URLEncodedUtils.parse(queryParamBuilder.toString(), Charset.forName("UTF-8")); - - Map<String, String> map = httpClientService.httpGet(queryURL + URLEncodedUtils.format(values, "utf-8"), 0); - if (Integer.parseInt(map.get("status")) == ResultStatusEnum.SUCCESS.getCode()) { - return BaseResultGenerator.generate(ResultStatusEnum.ACCEPTED.getCode(), null, "ok", null, null, null, null); + .append("database=").append(clickHouseHttpSource.getDbName()); + List<NameValuePair> values = URLEncodedUtils.parse(queryParamBuilder.toString(), StandardCharsets.UTF_8); + + HttpResponseResult responseResult = httpClientService.post(queryURL + URLEncodedUtils.format(values, StandardCharsets.UTF_8.toString()), killSql); + if (responseResult.getStatusCode() == HttpStatusCodeEnum.SUCCESS.getCode()) { + return BaseResultGenerator.generate(HttpStatusCodeEnum.ACCEPTED.getCode(), null, true, "ok", null, null, null, null); } else { - return BaseResultGenerator.failure(Integer.parseInt(map.get("status")), map.get("result")); + return BaseResultGenerator.failure(responseResult.getStatusCode(), responseResult.getErrorMessage()); } } @Override public BaseResult getProcesses(String queryId) { List<String> tableNames = Lists.newArrayList(); - SQLQuerySource sqlQuerySource = new SQLQuerySource(); - String tableName = "processes"; + SelectStatement selectStatement = new SelectStatement(); + String tableName = "processes_cluster"; tableNames.add(tableName); - String sql = StrUtil.format("select elapsed,total_rows_approx,read_rows from system.{} where query_id='{}'", tableName, queryId); - param.setQuery(sql); - sqlQuerySource.setExecSQL(param.getQuery()); - sqlQuerySource.setTableNames(tableNames); - param.setDbQuerySource(sqlQuerySource); + String sql = StrUtil.format("select elapsed,total_rows_approx,read_rows from system.{} where is_initial_query = 1 and query_id='{}'", tableName, queryId); + sqlQueryContext.setOriginalSQL(sql); + selectStatement.setExecSQL(sqlQueryContext.getOriginalSQL()); + selectStatement.setTableNames(tableNames); + sqlQueryContext.setDbSelectStatement(selectStatement); BaseResult baseResult = generateBaseResult(sql, Optional.of("ok")); Map<String, Object> result = new HashMap<>(16); if (!baseResult.isSuccess()) { @@ -220,6 +215,10 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { if (matcher.find()) { type = matcher.group(1); } + matcher = pDateTime.matcher(type); + if (matcher.find()) { + type = matcher.group(1); + } x.put("type", metaMap.get(type) == null ? DataTypeMapping.STRING : metaMap.get(type)); }); return metaList; @@ -238,7 +237,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { * 1.不支持 count(1) 需转为count(*) * 2.针对非聚合SQL，非分组sql in/not in 替换为 Global in/not * - * @return + * @return Converter */ public Converter generateStandard() { sql = generateCount(sql); @@ -285,7 +284,10 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { } public Converter generateDateFunction() { - sql = SQLFunctionUtil.generateDateFunction(sql, DBTypeEnum.CLICKHOUSE.getValue()); + sql = SQLFunctionUtil.generateDateFunction(sql, DBEngineType.CLICKHOUSE.getValue()); + if (sqlQueryContext.getDbSelectStatement().getUdfSet().stream().noneMatch(udf -> udf instanceof ROLLUP)) { + sql = SQLFunctionUtil.replaceRollup(sql); + } return this; } @@ -331,19 +333,19 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { * @return */ public Optimizer generateQueryLimit() { - if (StringUtil.isEmpty(param.getEngineQuerySource())) { - setDefaultResultRows(param.getDbQuerySource(), engineConfigSource.getDefaultResultNum()); - setMaxCacheResultRows(param.getDbQuerySource(), engineConfigSource.getMaxCacheNum()); + if (StringUtil.isEmpty(sqlQueryContext.getFederationSelectStatement())) { + setDefaultResultRows(sqlQueryContext.getDbSelectStatement(), engineConfigSource.getDefaultResultNum()); + setMaxCacheResultRows(sqlQueryContext.getDbSelectStatement(), engineConfigSource.getMaxCacheNum()); } else { - setDefaultResultRows(param.getEngineQuerySource(), engineConfigSource.getDefaultResultNum()); - setMaxCacheResultRows(param.getDbQuerySource(), engineConfigSource.getMaxCacheNum()); - setMaxCacheResultRows(param.getEngineQuerySource(), engineConfigSource.getMaxCacheNum()); + setDefaultResultRows(sqlQueryContext.getFederationSelectStatement(), engineConfigSource.getDefaultResultNum()); + setMaxCacheResultRows(sqlQueryContext.getDbSelectStatement(), engineConfigSource.getMaxCacheNum()); + setMaxCacheResultRows(sqlQueryContext.getFederationSelectStatement(), engineConfigSource.getMaxCacheNum()); } return this; } - private void setDefaultResultRows(SQLQuerySource sqlQuerySource, int defaultResultRows) { + private void setDefaultResultRows(SelectStatement sqlQuerySource, int defaultResultRows) { if (StringUtil.isBlank(sqlQuerySource.getLimit())) { sqlQuerySource.setSqlBody(sqlQuerySource.getSqlBody() + " limit " + defaultResultRows); sqlQuerySource.setLimit(String.valueOf(defaultResultRows)); @@ -357,7 +359,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { * @param sqlQuerySource * @return */ - private void setMaxCacheResultRows(SQLQuerySource sqlQuerySource, int maxCacheResultRows) { + private void setMaxCacheResultRows(SelectStatement sqlQuerySource, int maxCacheResultRows) { if (!sqlQuerySource.isEnableLimit()) { return; } @@ -382,8 +384,8 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { } public Optimizer generateSQL() { - SQLQuerySource sqlQuerySource = param.getDbQuerySource(); - sql = sqlQuerySource.getSqlBody(); + SelectStatement dbSelectStatement = sqlQueryContext.getDbSelectStatement(); + sql = dbSelectStatement.getSqlBody(); return this; } @@ -405,8 +407,8 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { sql = buffer.toString(); } catch (JSQLParserException | RuntimeException e) { log.error("SQL Rewrite add tableName and Alias Error: ", e); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - QGWErrorCode.SQL_BUILDER_EXCEPTION.getCode(), String.format(QGWErrorCode.SQL_BUILDER_EXCEPTION.getMessage(), e.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.SQL_REWRITE_AND_TRANSFORMATION_EXCEPTION.getCode(), String.format(CommonErrorCode.SQL_REWRITE_AND_TRANSFORMATION_EXCEPTION.getMessage(), e.getMessage())); } return this; } @@ -418,7 +420,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { */ public Optimizer generateUniq() { Matcher m = pUniq.matcher(sql); - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); while (m.find()) { if (m.groupCount() == 1) { String replaceValue = m.group(1); @@ -486,10 +488,10 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { */ public Optimizer generateSubQuery() { - if (StringUtil.isNotEmpty(param.getDbQuerySource().getJoins()) || param.getDbQuerySource().getTableNames().size() > 1) { + if (StringUtil.isNotEmpty(sqlQueryContext.getDbSelectStatement().getJoins()) || sqlQueryContext.getDbSelectStatement().getTableNames().size() > 1) { return this; } - if (StringUtil.isEmpty(param.getDbQuerySource().getWhereExpression())) { + if (StringUtil.isEmpty(sqlQueryContext.getDbSelectStatement().getWhereExpression())) { return this; } if (containAggregateFunction(sql) || containGroupBy(sql) || containDistinct(sql)) { @@ -532,21 +534,21 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { * @return */ private String getAvailableIndexTable() throws JSQLParserException { - String str = metadataService.getValueByKeyInSchemaDoc(param.getDbQuerySource().getTableNames().get(0), "index_table"); + String str = databaseService.getValueByKeyInSchemaDoc(sqlQueryContext.getDbSelectStatement().getTableNames().get(0), "index_table"); String[] indexTables = StringUtil.split(str, ","); - if (StringUtil.isEmpty(indexTables) || StringUtil.isEmpty(metadataService.getIndexKey(param.getDbQuerySource().getTableNames().get(0)))) { + if (StringUtil.isEmpty(indexTables) || StringUtil.isEmpty(databaseService.getIndexKey(sqlQueryContext.getDbSelectStatement().getTableNames().get(0)))) { return StringUtil.EMPTY; } - Expression expression = CondExpressionHelper.rewireCondExprBaseOnCK(param.getDbQuerySource().getWhereExpression().toString(), param.getDbQuerySource().getAliasFields(), false, null); + Expression expression = CondExpressionHelper.rewireCondExprBaseOnCK(sqlQueryContext.getDbSelectStatement().getWhereExpression().toString(), sqlQueryContext.getDbSelectStatement().getAliasFields(), false, null); List<String> actualFieldOfWhere = CondExpressionHelper.getDistinctFields(expression.toString()); table: for (String tempIndexTable : indexTables) { - List<String> indexKey = metadataService.getIndexKey(tempIndexTable); + List<String> indexKey = databaseService.getIndexKey(tempIndexTable); //判断一: where 是否包含索引键 if (StringUtil.isEmpty(indexKey) || !actualFieldOfWhere.contains(indexKey.get(0))) { continue; } - Schema indexTableSchema = metadataService.getSchemaByName(tempIndexTable); + LinkedHashMap<String, Object> indexTableSchema = databaseService.getSchemaByName(tempIndexTable); //判断二: where 条件列是否都在索引表中 for (String field : actualFieldOfWhere) { if (!isFieldOfSchema(indexTableSchema, field)) { @@ -557,7 +559,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { if (!isValidOfOrderElementsInSchema(indexTableSchema)) { continue; } - return indexTableSchema.getName(); + return String.valueOf(indexTableSchema.get("name")); } return StringUtil.EMPTY; } @@ -570,25 +572,25 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { */ private String getSubQueryByIndexTable(String indexTable) throws JSQLParserException { - String primaryKey = metadataService.getValueByKeyInSchemaDoc(param.getDbQuerySource().getTableNames().get(0), "primary_key"); - List<String> indexKey = metadataService.getIndexKey(param.getDbQuerySource().getTableNames().get(0)); + String primaryKey = databaseService.getValueByKeyInSchemaDoc(sqlQueryContext.getDbSelectStatement().getTableNames().get(0), "primary_key"); + List<String> indexKey = databaseService.getIndexKey(sqlQueryContext.getDbSelectStatement().getTableNames().get(0)); indexKey = indexKey.subList(0, indexKey.contains(primaryKey) ? indexKey.indexOf(primaryKey) + 1 : indexKey.size()); List<String> indexKeyMaster = Lists.newArrayList(); - String nameMaster = metadataService.getSchemaByName(param.getDbQuerySource().getTableNames().get(0)).getName(); + String nameMaster = String.valueOf(databaseService.getSchemaByName(sqlQueryContext.getDbSelectStatement().getTableNames().get(0)).get("name")); indexKey.forEach(o -> indexKeyMaster.add(nameMaster.concat(".").concat(o))); String subQuery = getSubquerySql(String.join(",", indexKey), indexTable); StringBuffer sb = new StringBuffer(); - sb.append("select ").append(param.getDbQuerySource().getSelect()).append(" from ").append(param.getDbQuerySource().getTableNames().get(0)).append(" where ( ").append(String.join(",", indexKeyMaster)).append(" ) in ( ").append(subQuery).append(")"); - if (StringUtil.isNotEmpty(param.getDbQuerySource().getWhereExpression())) { - sb.append(" and (").append(param.getDbQuerySource().getWhereExpression().toString()).append(")"); + sb.append("select ").append(sqlQueryContext.getDbSelectStatement().getSelect()).append(" from ").append(sqlQueryContext.getDbSelectStatement().getTableNames().get(0)).append(" where ( ").append(String.join(",", indexKeyMaster)).append(" ) in ( ").append(subQuery).append(")"); + if (StringUtil.isNotEmpty(sqlQueryContext.getDbSelectStatement().getWhereExpression())) { + sb.append(" and (").append(sqlQueryContext.getDbSelectStatement().getWhereExpression().toString()).append(")"); } - if (!param.getDbQuerySource().getOrderByElements().isEmpty()) { - sb.append(" order by ").append(param.getDbQuerySource().getOrderBy()); + if (!sqlQueryContext.getDbSelectStatement().getOrderByElements().isEmpty()) { + sb.append(" order by ").append(sqlQueryContext.getDbSelectStatement().getOrderBy()); } - if (StringUtil.isNotBlank(param.getDbQuerySource().getLimit())) { - sb.append(" limit ").append(param.getDbQuerySource().getLimit()); + if (StringUtil.isNotBlank(sqlQueryContext.getDbSelectStatement().getLimit())) { + sb.append(" limit ").append(sqlQueryContext.getDbSelectStatement().getLimit()); } return sb.toString(); } @@ -600,18 +602,18 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { * @return */ private String getSubQueryByPartitionKey(String sql) throws JSQLParserException { - String partitionKey = metadataService.getPartitionKey(param.getDbQuerySource().getTableNames().get(0)); + String partitionKey = databaseService.getPartitionKey(sqlQueryContext.getDbSelectStatement().getTableNames().get(0)); if (StringUtil.isBlank(partitionKey)) { - log.warn("Partition Key is Empty. Focus in tableName {} schema", param.getDbQuerySource().getTableNames().get(0)); + log.warn("Partition Key is Empty. Focus in tableName {} schema", sqlQueryContext.getDbSelectStatement().getTableNames().get(0)); return sql; } - String subQuery = getSubquerySql(partitionKey, param.getDbQuerySource().getTableNames().get(0)); + String subQuery = getSubquerySql(partitionKey, sqlQueryContext.getDbSelectStatement().getTableNames().get(0)); subQuery = subQuery.replaceAll("\\\\", "$0$0"); subQuery = subQuery.replaceAll("\\$", "\\\\\\$"); Matcher m = pWhere.matcher(sql); StringBuffer sb = new StringBuffer(); while (m.find()) { - m.appendReplacement(sb, "where " + metadataService.getSchemaByName(param.getDbQuerySource().getTableNames().get(0)).getName().concat(".").concat(partitionKey) + " in ( " + subQuery + ") and "); + m.appendReplacement(sb, "where " + String.valueOf(databaseService.getSchemaByName(sqlQueryContext.getDbSelectStatement().getTableNames().get(0)).get("name")).concat(".").concat(partitionKey) + " in ( " + subQuery + ") and "); } m.appendTail(sb); return sb.toString(); @@ -628,19 +630,16 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { StringBuilder subQuery = new StringBuilder(); subQuery.append("select " + columnName + " from ").append(tableName) - .append(" where ").append(CondExpressionHelper.rewireCondExprBaseOnCK(param.getDbQuerySource().getWhereExpression().toString(), param.getDbQuerySource().getAliasFields(), true, tableName)); + .append(" where ").append(CondExpressionHelper.rewireCondExprBaseOnCK(sqlQueryContext.getDbSelectStatement().getWhereExpression().toString(), sqlQueryContext.getDbSelectStatement().getAliasFields(), true, tableName)); //order by中元素存在别名时做替换处理 - List<OrderByElement> listOrderElement = param.getDbQuerySource().getOrderByElements(); + List<OrderByElement> listOrderElement = sqlQueryContext.getDbSelectStatement().getOrderByElements(); if (StringUtil.isNotEmpty(listOrderElement)) { - String orderBy = param.getDbQuerySource().getOrderBy(); + String orderBy = sqlQueryContext.getDbSelectStatement().getOrderBy(); for (OrderByElement order : listOrderElement) { String orderField = order.getExpression().toString(); - if ((orderField.startsWith("\"") && orderField.endsWith("\"")) - || (orderField.startsWith("`") && orderField.endsWith("`"))) { - orderField = orderField.substring(1, orderField.length() - 1); - } - String originalField = param.getDbQuerySource().getAliasFields().get(orderField); + orderField = SQLHelper.removeQuotesAndBackticks(orderField); + String originalField = sqlQueryContext.getDbSelectStatement().getAliasFields().get(orderField); if (StringUtil.isNotBlank(originalField)) { orderBy = orderBy.replaceAll("(\"" + orderField + "\")|(\\b" + orderField + "\\b)", originalField); } @@ -650,13 +649,13 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { } - if (StringUtil.isNotBlank(param.getDbQuerySource().getLimit())) { - String[] limits = param.getDbQuerySource().getLimit().split(","); + if (StringUtil.isNotBlank(sqlQueryContext.getDbSelectStatement().getLimit())) { + String[] limits = sqlQueryContext.getDbSelectStatement().getLimit().split(","); if (limits.length == 2) { int rows = Integer.valueOf(limits[0].trim()) + Integer.valueOf(limits[1].trim()); subQuery.append(" limit ").append(rows); } else { - subQuery.append(" limit ").append(param.getDbQuerySource().getLimit()); + subQuery.append(" limit ").append(sqlQueryContext.getDbSelectStatement().getLimit()); } } else { @@ -668,21 +667,21 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { /** * 判断order by 元素是否属于schema字段中 * - * @param schema + * @param schemaMap * @return */ - private boolean isValidOfOrderElementsInSchema(Schema schema) throws JSQLParserException { - List<OrderByElement> listOrderElement = param.getDbQuerySource().getOrderByElements(); + private boolean isValidOfOrderElementsInSchema(Map<String, Object> schemaMap) throws JSQLParserException { + List<OrderByElement> listOrderElement = sqlQueryContext.getDbSelectStatement().getOrderByElements(); if (StringUtil.isEmpty(listOrderElement)) { return true; } for (OrderByElement order : listOrderElement) { OrderByElement orderByElement = new OrderByElement(); - Expression expression = CondExpressionHelper.rewireCondExprBaseOnCK(String.valueOf(order.getExpression()), param.getDbQuerySource().getAliasFields(), false, null); + Expression expression = CondExpressionHelper.rewireCondExprBaseOnCK(String.valueOf(order.getExpression()), sqlQueryContext.getDbSelectStatement().getAliasFields(), false, null); orderByElement.setExpression(expression); List<String> fields = CondExpressionHelper.getDistinctFieldsOfOrderBy(orderByElement); for (String field : fields) { - if (!isFieldOfSchema(schema, field)) { + if (!isFieldOfSchema(schemaMap, field)) { return false; } } @@ -693,16 +692,18 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { /** * 判断schema中是否存在field列 * - * @param schema + * @param schemaMap * @param fieldName * @return */ - private boolean isFieldOfSchema(Schema schema, String fieldName) { - Schema.Field field = schema.getField(fieldName); - if (StringUtil.isEmpty(field)) { - return false; + private boolean isFieldOfSchema(Map<String, Object> schemaMap, String fieldName) { + List<Map> fields = (List<Map>) schemaMap.get("fields"); + for (Map next : fields) { + if (next.get("name").equals(fieldName)) { + return true; + } } - return true; + return false; } public String build() { @@ -724,23 +725,27 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { @Override public BaseResult generateBaseResult(String sql, Optional<String> message) { - Map<String, String> results = executeHttpPost(sql); - int status = Integer.parseInt(results.get("status")); - if (status == ResultStatusEnum.SUCCESS.getCode()) { + HttpResponseResult responseResult = executeHttpPost(sql); + if (responseResult.getStatusCode() == HttpStatusCodeEnum.SUCCESS.getCode()) { baseResult = BaseResultGenerator.generate( - Integer.parseInt(results.get("status")), + responseResult.getStatusCode(), ResultCodeEnum.SUCCESS.getCode(), + true, message.get(), - JSON.parseObject(results.get("result"), Map.class), + JSON.parseObject(responseResult.getResponseBody(), Map.class), null, null, - QueryFormatEnum.JSON.getValue()); + OutputMode.JSON.getValue()); } else { - baseResult = BaseResultGenerator.generate(Integer.parseInt(results.get("status")), - - ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), message.get()), - StringUtil.isNotEmpty(results.get("message")) ? results.get("message") : results.get("result"), null, null, param.getFormat()); + baseResult = BaseResultGenerator.generate(responseResult.getStatusCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + false, + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), message.get()), + responseResult.getErrorMessage(), + null, + null, + sqlQueryContext.getFormat()); } return build(); } @@ -759,15 +764,16 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { statistics.put("rows_read", statisticMap.get("rows_read")); statistics.put("bytes_read", statisticMap.get("bytes_read")); baseResult.setData(resultMap.get("data")); - baseResult.setMeta(unifyMeta(resultMap.get("meta"))); + baseResult.setMeta(ColumnCategoryHelper.expandMetaCategory(unifyMeta(resultMap.get("meta")), sqlQueryContext.getDbSelectStatement(), sqlQueryContext.getDbEngine())); baseResult.setStatistics(statistics); } else { - if (baseResult.getStatus() >= ResultStatusEnum.BAD_REQUEST.getCode() && baseResult.getStatus() < ResultStatusEnum.SERVER_ERROR.getCode()) { - throw new QGWBusinessException(baseResult.getStatus(), QGWErrorCode.SQL_EXECUTION_BAD_REQUEST_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_BAD_REQUEST_EXCEPTION.getMessage(), simplyErrorMessage(dataObject))); + if (baseResult.getStatus() >= HttpStatusCodeEnum.BAD_REQUEST.getCode() + && baseResult.getStatus() < HttpStatusCodeEnum.SERVER_ERROR.getCode()) { + throw new QGWBusinessException(baseResult.getStatus(), CommonErrorCode.BAD_REQUEST_SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_EXECUTION_EXCEPTION.getMessage(), simplyErrorMessage(dataObject))); } - throw new QGWBusinessException(baseResult.getStatus(), QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getMessage(), simplyErrorMessage(dataObject))); + throw new QGWBusinessException(baseResult.getStatus(), CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(), simplyErrorMessage(dataObject))); } return baseResult; } @@ -787,6 +793,61 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { return StrUtil.removeAllLineBreaks(message).trim(); } + @SneakyThrows + @Override + List<Map<String, String>> buildExampleMeta() { + if (baseResult.getMeta() == null) { + return Lists.newArrayList(); + } + List<Map<String, String>> metaList = (List<Map<String, String>>) baseResult.getMeta(); + Map<String, SelectItemHelper.AliasObject> selectItem = SelectItemHelper.getSelectItem(sqlQueryContext.getDbSelectStatement().getSqlBody()); + for (Map<String, String> meta : metaList) { + toPerfectMeta(selectItem, meta); + } + return metaList; + } + + private void toPerfectMeta(Map<String, SelectItemHelper.AliasObject> aliasObjects, Map<String, String> meta) { + if (aliasObjects == null) { + return; + } + String name = meta.get(MetaConst.META_NAME); + SelectItemHelper.AliasObject aliasObject = aliasObjects.get(name); + if (aliasObject == null) { + return; + } + if (aliasObject instanceof SelectItemHelper.AliasColumn) { + SelectItemHelper.AliasColumn aliasColumn = (SelectItemHelper.AliasColumn) aliasObject; + String fieldName = aliasColumn.getFieldName(); + meta.put(MetaConst.META_FIELD_NAME, fieldName); + Map schemaInfo = databaseService.getSchemaInfo(MetadataType.FIELDS.getValue(), sqlQueryContext.getDbSelectStatement().getTableNames().get(0), false); + Object dateType = JSONPath.extract(JSON.toJSONString(schemaInfo), "$.fields[?(@.name == \"" + fieldName + "\")].doc.constraints.type"); + if (dateType instanceof List && ((List<?>) dateType).size() > 0) { + dateType = ((List<?>) dateType).get(0); + meta.put(MetaConst.META_DATA_TYPE, dateType.toString()); + } + } else if (aliasObject instanceof SelectItemHelper.AliasFunExpr) { + SelectItemHelper.AliasFunExpr aliasDatetime = (SelectItemHelper.AliasFunExpr) aliasObject; + String dateType = aliasDatetime.getDateType(); + if (dateType != null) { + meta.put(MetaConst.META_DATA_TYPE, dateType); + } + } + } + + @Override + List<Object> buildExampleData() { + if (baseResult.getMeta() == null) { + return Lists.newArrayList(); + } + List<Map<String, String>> metaList = (List<Map<String, String>>) baseResult.getMeta(); + Map<String, List<Object>> schemaDataDict = databaseService.getSchemaDataDict(sqlQueryContext.getDbSelectStatement().getTableNames().get(0)); + Long rowCount = SQLHelper.getRowCount(sqlQueryContext.getDbSelectStatement().getSqlBody()); + String queryType = QueryTypeHelper.determineQueryType(sqlQueryContext.getDbSelectStatement()); + Object param = QueryTypeHelper.determineParam(sqlQueryContext.getDbSelectStatement(), queryType); + return ExampleDataHelper.buildExampleData(queryType, metaList, rowCount, schemaDataDict, param); + } + private boolean containAggregateFunction(String sql) { return (SQLFunctionUtil.pAggregateFunStandard.matcher(sql).find() || pAggregateFunSpecificOfCK.matcher(sql).find()); @@ -802,20 +863,14 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { /** - * 通过递归方式将原SQL转换为样例SQL - * 做数据集限制: - * 1. 将最内层sql的表名, 替换成(select * from tableName limit XXX) as alias - * 2. 可对where、having添加：OR 1=1 + * 将目标SQL转换为Dry Run SQ。使其不扫描数据，仅做执行 * - * @param sql 原始sql - * @return 经转换后的样例SQL + * @param sql + * @return * @throws JSQLParserException */ private String convertSampleSql(String sql) throws JSQLParserException { - boolean isSampled = BooleanUtil.isTrue(param.getSampled()); - String dataSet = StrUtil.format("SELECT * FROM {}", - isSampled ? " {} WHERE rand() % 100 = 0 LIMIT 1000 " : " {} LIMIT 0 "); - return new ClickhouseSampleSQLHelper(sql, dataSet, isSampled).build(); + return new SampleSQLHelper(sql, " 1 != 1 ", false, null).build(); } /** @@ -824,38 +879,54 @@ public class ClickHouseDialect extends AbstractDataSourceDialect { * @param sql SQL语句 * @return */ - private Map<String, String> executeHttpPost(String sql) { + private HttpResponseResult executeHttpPost(String sql) { String queryURL = URLUtil.normalize(clickHouseHttpSource.getUrl() + "/?"); - ClickHouseHttpQuery clickHouseHttpQuery = buildHttpQueryParameter(); - log.info("DB engine is :{},execute http query is :{}", DBTypeEnum.CLICKHOUSE.getValue(), queryURL + - Encodes.urlDecode(clickHouseHttpQuery.getQueryParameter()).replaceAll("password=(.*?)&", "password=XXX&") + ". body sql is:" + sql); - List<NameValuePair> values = URLEncodedUtils.parse(clickHouseHttpQuery.getQueryParameter(), Charset.forName("UTF-8")); - return httpClientService.httpPost(queryURL + URLEncodedUtils.format(values, "utf-8"), sql.concat(" FORMAT ").concat(QueryFormatEnum.JSON.getValue().toUpperCase()) - , clickHouseHttpQuery.getSocketTimeOut()); + String queryParameter = enrichContextAndBuildQueryParameter(); + log.info("Query ID: {}, SQL: {}", sqlQueryContext.getQueryId(), sql); + if (log.isDebugEnabled()) { + log.debug("DB engine is: {}, : {}", DBEngineType.CLICKHOUSE.getValue(), queryURL + + Encodes.urlDecode(queryParameter).replaceAll("password=(.*?)&", "password=XXX&") + ". body sql is:" + sql); + } + List<NameValuePair> values = URLEncodedUtils.parse(queryParameter, StandardCharsets.UTF_8); + long start = System.currentTimeMillis(); + HttpResponseResult responseResult = httpClientService.post(queryURL + URLEncodedUtils.format(values, "utf-8"), + sqlQueryContext.getTimeout(), + sql.concat(" FORMAT ").concat(OutputMode.JSON.getValue().toUpperCase()).concat(" settings output_format_json_quote_64bit_integers=0")); + long elapsed = System.currentTimeMillis() - start; + if (elapsed > engineConfigSource.getHighLatencyThreshold()) { + log.warn("DB engine is: {}, queryId: {}, elapsed: {} ms, sql: {}", DBEngineType.CLICKHOUSE.getValue(), sqlQueryContext.getQueryId(), elapsed, sql); + } + return responseResult; } - private ClickHouseHttpQuery buildHttpQueryParameter() { - ClickHouseHttpQuery clickHouseHttpQuery = new ClickHouseHttpQuery(); + private String enrichContextAndBuildQueryParameter() { StringBuilder queryParameterBuilder = new StringBuilder(1024); - String dbName = StringUtil.isNotEmpty(param.getDbQuerySource().getTableNames()) - ? metadataService.getDBNameByTableName(param.getDbQuerySource().getTableNames().get(0)) : clickHouseHttpSource.getDbName(); + + String dbName = StringUtil.isNotEmpty(sqlQueryContext.getDbSelectStatement().getTableNames()) + ? databaseService.getDBNameByTableName(sqlQueryContext.getDbSelectStatement().getTableNames().get(0)) : clickHouseHttpSource.getDbName(); + if (dbName.equalsIgnoreCase(clickHouseHttpSource.getSystemDBName())) { - param.setOption(QueryOptionEnum.LONG_TERM.getValue()); + sqlQueryContext.setOption(QueryOption.LONG_TERM.getValue()); + } + + int timeout = (QueryOption.LONG_TERM.getValue().equalsIgnoreCase(sqlQueryContext.getOption())) ? + httpConfig.getCkLongTermAccountSocketTimeOut() : httpConfig.getCkRealTimeAccountSocketTimeOut(); + sqlQueryContext.setTimeout(timeout); + + if (StringUtil.isBlank(sqlQueryContext.getQueryId())) { + sqlQueryContext.setQueryId(DigestUtil.md5Hex(StringUtil.createUUID())); } - if (QueryOptionEnum.LONG_TERM.getValue().equalsIgnoreCase(param.getOption())) { - clickHouseHttpQuery.setSocketTimeOut(httpConfig.getCkLongTermAccountSocketTimeOut()); + //QueryContext set queryParameter + if (QueryOption.LONG_TERM.getValue().equalsIgnoreCase(sqlQueryContext.getOption())) { queryParameterBuilder.append(getLongTermAccount()).append("&") .append("database=").append(dbName); } else { - clickHouseHttpQuery.setSocketTimeOut(httpConfig.getCkRealTimeAccountSocketTimeOut()); queryParameterBuilder.append(getRealTimeAccount()).append("&") .append("database=").append(dbName); } - queryParameterBuilder.append("&").append("query_id=") - .append(StringUtil.isNotBlank(param.getQueryId()) ? param.getQueryId() : DigestUtil.md5Hex(StringUtil.createUUID())); - clickHouseHttpQuery.setQueryParameter(queryParameterBuilder.toString()); - return clickHouseHttpQuery; + queryParameterBuilder.append("&").append("query_id=").append(sqlQueryContext.getQueryId()); + return queryParameterBuilder.toString(); } private String getRealTimeAccount() { diff --git a/src/main/java/com/mesalab/qgw/dialect/Dialect.java b/src/main/java/com/mesalab/qgw/dialect/Dialect.java index 21e6903b..5e0eb02b 100644 --- a/src/main/java/com/mesalab/qgw/dialect/Dialect.java +++ b/src/main/java/com/mesalab/qgw/dialect/Dialect.java @@ -16,7 +16,7 @@ public interface Dialect { * 执行SQL语法检查 * @return */ - BaseResult executeSyntaxCheck(); + BaseResult executeSyntaxValidation(); /** * 执行查询操作 diff --git a/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java b/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java index 9d260fd0..76ce1ed1 100644 --- a/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java +++ b/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java @@ -1,67 +1,92 @@ package com.mesalab.qgw.dialect; -import cn.hutool.core.util.BooleanUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.URLUtil; import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONPath; import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.JsonPathException; -import com.mesalab.common.utils.sqlparser.DruidSampleSQLHelper; +import com.mesalab.common.utils.sqlparser.*; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; import com.mesalab.common.entity.DataTypeMapping; import com.mesalab.common.enums.*; import com.mesalab.common.utils.*; -import com.mesalab.common.utils.sqlparser.SQLFunctionUtil; -import com.mesalab.common.utils.sqlparser.SQLHelper; +import com.mesalab.qgw.constant.MetaConst; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; +import com.mesalab.common.exception.CommonErrorCode; import com.mesalab.qgw.model.basic.*; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.service.impl.HttpClientService; +import com.mesalab.qgw.model.basic.SQLQueryContext; import com.geedgenetworks.utils.StringUtil; +import com.mesalab.qgw.service.DatabaseService; +import com.mesalab.qgw.service.impl.HttpClientServiceV2; import lombok.Data; +import lombok.SneakyThrows; import net.sf.jsqlparser.JSQLParserException; -import org.apache.commons.compress.utils.Lists; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + public class DruidDialect extends AbstractDataSourceDialect { private static final Log log = LogFactory.get(); + public static final Pattern pAggregateFunSpecificOfDruid = Pattern.compile("\\b(QUANTILE_HDR|HDR_HISTOGRAM|HLLD|APPROX_COUNT_DISTINCT_HLLD|PERCENTILES_HDR)\\s*\\(", Pattern.CASE_INSENSITIVE); private DruidIoHttpSource druidIoHttpSource = (DruidIoHttpSource) SpringContextUtil.getBean("druidIoHttpSource"); - private HttpClientService httpClientService = (HttpClientService) SpringContextUtil.getBean("httpClientService"); + private HttpClientServiceV2 httpClientService = (HttpClientServiceV2) SpringContextUtil.getBean("httpClientServiceV2"); private EngineConfigSource engineConfigSource = (EngineConfigSource) SpringContextUtil.getBean("engineConfigSource"); private HttpConfig httpConfig = (HttpConfig) SpringContextUtil.getBean("httpConfig"); + private DatabaseService databaseService = (DatabaseService) SpringContextUtil.getBean("databaseService"); private int pageOffset; private static final String CLAZZ_NAME_CALCITE_EXCEPTION = "com.zdjizhi.calcite.runtime.CalciteContextException: "; private Stopwatch watch = Stopwatch.createUnstarted(); - public QueryProfile param; + public BaseResult baseResult; - public DruidDialect(QueryProfile param) { - this.param = param; + private final static Map<String, String> metaMap = Maps.newHashMap(); + + static { + metaMap.put("CHAR", DataTypeMapping.STRING); + metaMap.put("VARCHAR", DataTypeMapping.STRING); + metaMap.put("FLOAT", DataTypeMapping.FLOAT); + metaMap.put("DECIMAL", DataTypeMapping.DOUBLE); + metaMap.put("REAL", DataTypeMapping.DOUBLE); + metaMap.put("DOUBLE", DataTypeMapping.DOUBLE); + metaMap.put("BOOLEAN", DataTypeMapping.BOOLEAN); + metaMap.put("TINYINT", DataTypeMapping.INT); + metaMap.put("SMALLINT", DataTypeMapping.INT); + metaMap.put("INTEGER", DataTypeMapping.INT); + metaMap.put("BIGINT", DataTypeMapping.LONG); + metaMap.put("TIMESTAMP", DataTypeMapping.DATE); + metaMap.put("DATE", DataTypeMapping.DATE); + + metaMap.put("ARRAY", null); + metaMap.put("OTHER", null); + } + + public DruidDialect(SQLQueryContext queryContext) { + super(queryContext); } @Override void init() { - if (StringUtil.isNotEmpty(param.getDbQuerySource())) { - param.getDbQuerySource().setExecSQL(param.getDbQuerySource().getSqlBody()); + if (StringUtil.isNotEmpty(sqlQueryContext.getDbSelectStatement())) { + sqlQueryContext.getDbSelectStatement().setExecSQL(sqlQueryContext.getDbSelectStatement().getSqlBody()); } else { - SQLQuerySource sqlQuerySource = new SQLQuerySource(); - sqlQuerySource.setExecSQL(param.getQuery()); - param.setDbQuerySource(sqlQuerySource); + SelectStatement sqlQuerySource = new SelectStatement(); + sqlQuerySource.setExecSQL(sqlQueryContext.getOriginalSQL()); + sqlQueryContext.setDbSelectStatement(sqlQuerySource); } baseResult = BaseResultGenerator.success(); } - @Override public boolean supportsLimit() { return true; @@ -70,42 +95,38 @@ public class DruidDialect extends AbstractDataSourceDialect { @Override public String convertQuery(String sql) { - return new Converter(sql).generateStandard().generateDateFunction().build(); + return new Converter(sql).generateStandard().mergeFunctions().generateDateFunction().build(); } @Override - public String getBestQuery() { + public String generateRuleBaseOptimizedQuery() { return new Optimizer().generateQueryLimit().generateSQL().build(); } - @Override - public String getOriginalQuery() { - return param.getQuery(); - } - /** * 获取语法检测sql * * @return */ + @SneakyThrows @Override public String getSampleQuery() { - String sampleSQL = param.getDbQuerySource().getSqlBody(); + String sampleSQL = sqlQueryContext.getDbSelectStatement().getSqlBody(); try { sampleSQL = convertQuery(convertSampleSql(sampleSQL)); } catch (JSQLParserException e) { log.error("syntax-check sql error {}, execute original sql: {}, error is: {} ", - sampleSQL, sampleSQL = param.getDbQuerySource().getSqlBody(), e.getMessage() == null ? e.getCause() : e.getMessage()); + sampleSQL, sampleSQL = sqlQueryContext.getDbSelectStatement().getSqlBody(), e.getMessage() == null ? e.getCause() : e.getMessage()); } return sampleSQL; } @Override String getExplainQuery() { - return "EXPLAIN PLAN FOR " + convertQuery(getBestQuery()); + return "EXPLAIN PLAN FOR " + convertQuery(generateRuleBaseOptimizedQuery()); } @Override @@ -113,12 +134,12 @@ public class DruidDialect extends AbstractDataSourceDialect { String url = druidIoHttpSource.getUrl(); url = url.substring(0, url.indexOf("/")); String deleteUrl = URLUtil.normalize(url).concat("/druid/v2/").concat(queryId); - Map<String, String> map = httpClientService.httpDelete(deleteUrl, 1000); - if (Integer.parseInt(map.get("status")) == ResultStatusEnum.SUCCESS.getCode() - || Integer.parseInt(map.get("status")) == ResultStatusEnum.ACCEPTED.getCode()) { - return BaseResultGenerator.generate(ResultStatusEnum.ACCEPTED.getCode(), null, "ok", null, null, null, null); + HttpResponseResult httpResponseResult = httpClientService.delete(deleteUrl); + if (httpResponseResult.getStatusCode() == HttpStatusCodeEnum.SUCCESS.getCode() + || httpResponseResult.getStatusCode() == HttpStatusCodeEnum.ACCEPTED.getCode()) { + return BaseResultGenerator.generate(HttpStatusCodeEnum.ACCEPTED.getCode(), null, true, "ok", null, null, null, null); } else { - return BaseResultGenerator.failure(Integer.parseInt(map.get("status")), map.get("result")); + return BaseResultGenerator.failure(httpResponseResult.getStatusCode(), httpResponseResult.getErrorMessage()); } } @@ -133,20 +154,37 @@ public class DruidDialect extends AbstractDataSourceDialect { * @param sql SQL语句 * @return */ - private Map<String, String> executeHttpPost(String sql) { + private HttpResponseResult executeHttpPost(String sql) { String queryURL = URLUtil.normalize(druidIoHttpSource.getUrl()); - DruidQueryParam druidQueryParam = getDruidQueryParam(sql); - log.info("DB engine is :{},execute query is: {}", DBTypeEnum.DRUID.getValue(), JSON.toJSONString(druidQueryParam)); - int socketTimeOut = httpConfig.getDruidSocketTimeOut(); - return httpClientService.httpPost(queryURL, JSON.toJSONString(druidQueryParam), socketTimeOut); + DruidQueryParam druidQueryParam = enrichContextAndBuildQueryParameter(sql); + log.info("Query ID: {}, SQL: {}", sqlQueryContext.getQueryId(), sql); + if (log.isDebugEnabled()) { + log.debug("DB engine is: {}, : {}", DBEngineType.DRUID.getValue(), JSON.toJSONString(druidQueryParam) + + ". body sql is:" + sql); + } + long start = System.currentTimeMillis(); + HttpResponseResult responseResult = httpClientService.post(queryURL, sqlQueryContext.getTimeout(), JSON.toJSONString(druidQueryParam)); + long elapsed = System.currentTimeMillis() - start; + if (elapsed > engineConfigSource.getHighLatencyThreshold()) { + log.warn("DB engine is: {}, queryId: {}, elapsed: {} ms, sql: {}", DBEngineType.DRUID.getValue(), sqlQueryContext.getQueryId(), elapsed, sql); + } + return responseResult; } - private DruidQueryParam getDruidQueryParam(String sql) { + private DruidQueryParam enrichContextAndBuildQueryParameter(String sql) { + sqlQueryContext.setTimeout(httpConfig.getDruidSocketTimeOut()); + if (StringUtil.isBlank(sqlQueryContext.getQueryId())) { + sqlQueryContext.setQueryId(DigestUtil.md5Hex(StringUtil.createUUID())); + } DruidQueryParam druidQueryParam = new DruidQueryParam(); druidQueryParam.setQuery(sql); druidQueryParam.getContext().put("skipEmptyBuckets", druidIoHttpSource.getSkipEmptyBuckets()); - druidQueryParam.getContext().put("sqlQueryId", StringUtil.isNotBlank(param.getQueryId()) ? param.getQueryId() : DigestUtil.md5Hex(StringUtil.createUUID())); + druidQueryParam.getContext().put("sqlQueryId", sqlQueryContext.getQueryId()); + druidQueryParam.getContext().put("maxSubqueryRows", druidIoHttpSource.getMaxSubqueryRows()); druidQueryParam.setResultFormat("object"); + druidQueryParam.setHeader(true); + druidQueryParam.setTypesHeader(true); + druidQueryParam.setSqlTypesHeader(true); return druidQueryParam; } @@ -154,17 +192,15 @@ public class DruidDialect extends AbstractDataSourceDialect { public BaseResult generateBaseResult(String sql, Optional<String> message) { watch.start(); - Map<String, String> results = executeHttpPost(sql); + HttpResponseResult responseResult = executeHttpPost(sql); - int status = Integer.parseInt(results.get("status")); - if (status == ResultStatusEnum.SUCCESS.getCode()) { - baseResult = BaseResultGenerator.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.SUCCESS.getCode(), message.get(), - (List<Object>) JSON.parseObject(results.get("result"), Object.class), null, null, QueryFormatEnum.JSON.getValue()); + if (responseResult.getStatusCode() == HttpStatusCodeEnum.SUCCESS.getCode()) { + baseResult = BaseResultGenerator.generate(responseResult.getStatusCode(), ResultCodeEnum.SUCCESS.getCode(), true, message.get(), + (List<Object>) JSON.parseObject(responseResult.getResponseBody(), Object.class), null, null, OutputMode.JSON.getValue()); } else { - baseResult = BaseResultGenerator.generate(Integer.parseInt(results.get("status")), - - ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), message.get()), - StringUtil.isNotEmpty(results.get("message")) ? results.get("message") : results.get("result"), null, null, param.getFormat()); + baseResult = BaseResultGenerator.generate(responseResult.getStatusCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), false, String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), message.get()), + responseResult.getErrorMessage(), null, null, sqlQueryContext.getFormat()); } return build(); } @@ -179,6 +215,8 @@ public class DruidDialect extends AbstractDataSourceDialect { baseResult.setMessage("ok"); } List<Object> allResults = (List<Object>) dataObject; + Object metaObj = allResults.remove(0); + baseResult.setMeta(ColumnCategoryHelper.expandMetaCategory(unifyMeta(JSON.parseObject(JSON.toJSONString(metaObj), Map.class)), sqlQueryContext.getDbSelectStatement(), sqlQueryContext.getDbEngine())); if (pageOffset > 0 && StringUtil.isNotEmpty(allResults)) { if (pageOffset >= allResults.size()) { allResults = new ArrayList<>(); @@ -190,15 +228,14 @@ public class DruidDialect extends AbstractDataSourceDialect { statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); statistics.put("rows_read", allResults.size()); baseResult.setData(allResults); - baseResult.setMeta(unifyMeta(allResults)); baseResult.setStatistics(statistics); } else { - if (baseResult.getStatus() >= ResultStatusEnum.BAD_REQUEST.getCode() && baseResult.getStatus() < ResultStatusEnum.SERVER_ERROR.getCode()) { - throw new QGWBusinessException(baseResult.getStatus(), QGWErrorCode.SQL_EXECUTION_BAD_REQUEST_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_BAD_REQUEST_EXCEPTION.getMessage(), simplyErrorMessage(dataObject))); + if (baseResult.getStatus() >= HttpStatusCodeEnum.BAD_REQUEST.getCode() && baseResult.getStatus() < HttpStatusCodeEnum.SERVER_ERROR.getCode()) { + throw new QGWBusinessException(baseResult.getStatus(), CommonErrorCode.BAD_REQUEST_SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_EXECUTION_EXCEPTION.getMessage(), simplyErrorMessage(dataObject))); } - throw new QGWBusinessException(baseResult.getStatus(), QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getMessage(), simplyErrorMessage(dataObject))); + throw new QGWBusinessException(baseResult.getStatus(), CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(), simplyErrorMessage(dataObject))); } return baseResult; } @@ -216,45 +253,87 @@ public class DruidDialect extends AbstractDataSourceDialect { return StrUtil.removeAllLineBreaks(message).trim(); } + @SneakyThrows + @Override + List<Map<String, String>> buildExampleMeta() { + if (baseResult.getMeta() == null) { + return Lists.newArrayList(); + } + List<Map<String, String>> metaList = (List<Map<String, String>>) baseResult.getMeta(); + Map<String, SelectItemHelper.AliasObject> selectItem = SelectItemHelper.getSelectItem(sqlQueryContext.getDbSelectStatement().getSqlBody()); + for (Map<String, String> meta : metaList) { + toPerfectMeta(selectItem, meta); + } + return metaList; + } + + @Override + public List<Object> buildExampleData() { + if (baseResult.getMeta() == null) { + return Lists.newArrayList(); + } + List<Map<String, String>> metaList = (List<Map<String, String>>) baseResult.getMeta(); + Map<String, List<Object>> schemaDataDict = databaseService.getSchemaDataDict(sqlQueryContext.getDbSelectStatement().getTableNames().get(0)); + Long rowCount = SQLHelper.getRowCount(sqlQueryContext.getDbSelectStatement().getSqlBody()); + String queryType = QueryTypeHelper.determineQueryType(sqlQueryContext.getDbSelectStatement()); + Object param = QueryTypeHelper.determineParam(sqlQueryContext.getDbSelectStatement(), queryType); + return ExampleDataHelper.buildExampleData(queryType, metaList, rowCount, schemaDataDict, param); + } /** - * 做时间范围条件过滤: 去除原有最内层条件, 重新添加时间范围 + * 将目标SQL转换为Dry Run SQ。使其不扫描数据，仅做执行 * * @param sql * @return * @throws JSQLParserException */ private String convertSampleSql(String sql) throws JSQLParserException { - boolean isSampled = BooleanUtil.isTrue(param.getSampled()); - String filterExpr = String.format("%s >= CURRENT_TIMESTAMP - INTERVAL %s AND %s < CURRENT_TIMESTAMP", - param.getDbQuerySource().getPartitionKey(), isSampled ? "'12' HOUR " : "'1' SECOND", param.getDbQuerySource().getPartitionKey()); - return new DruidSampleSQLHelper(sql, filterExpr, isSampled).build(); + return new SampleSQLHelper(sql, " 1 != 1 ", false, null).build(); } - private List<Map<String, String>> unifyMeta(List<Object> list) { - List<Map<String, String>> metaList = Lists.newArrayList(); - if (StringUtil.isEmpty(list)) { - return metaList; + private List<Map<String, String>> unifyMeta(Map<String, Map<String, String>> metaMap) { + List<Map<String, String>> result = new ArrayList<>(); + if (metaMap == null || metaMap.isEmpty()) { + return result; } - Map<String, Object> data = JsonPath.read(list.get(0), "$"); - for (String key : data.keySet()) { - Map<String, String> temp = Maps.newHashMap(); - temp.put("name", key); - Object value = data.get(key); - if (value instanceof Integer) { - temp.put("type", DataTypeMapping.INT); - } else if (value instanceof Long) { - temp.put("type", DataTypeMapping.LONG); - } else if (value instanceof Float) { - temp.put("type", DataTypeMapping.FLOAT); - } else if (value instanceof Double) { - temp.put("type", DataTypeMapping.DOUBLE); - } else { - temp.put("type", DataTypeMapping.STRING); + for (Map.Entry<String, Map<String, String>> entry : metaMap.entrySet()) { + String metaName = entry.getKey(); + Map<String, String> typeMap = entry.getValue(); + Map<String, String> meta = Maps.newHashMap(); + meta.put(MetaConst.META_NAME, metaName); + String type = DruidDialect.metaMap.get(typeMap.get("sqlType")); + meta.put(MetaConst.META_TYPE, type == null ? DataTypeMapping.STRING : type); + result.add(meta); + } + return result; + } + + private void toPerfectMeta(Map<String, SelectItemHelper.AliasObject> aliasObjects, Map<String, String> meta) { + if (aliasObjects == null) { + return; + } + String name = meta.get(MetaConst.META_NAME); + SelectItemHelper.AliasObject aliasObject = aliasObjects.get(name); + if (aliasObject == null) { + return; + } + if (aliasObject instanceof SelectItemHelper.AliasColumn) { + SelectItemHelper.AliasColumn aliasColumn = (SelectItemHelper.AliasColumn) aliasObject; + String fieldName = aliasColumn.getFieldName(); + meta.put(MetaConst.META_FIELD_NAME, fieldName); + Map schemaInfo = databaseService.getSchemaInfo(MetadataType.FIELDS.getValue(), sqlQueryContext.getDbSelectStatement().getTableNames().get(0), false); + Object dateType = JSONPath.extract(JSON.toJSONString(schemaInfo), "$.fields[?(@.name == \"" + fieldName + "\")].doc.constraints.type"); + if (dateType instanceof List && ((List<?>) dateType).size() > 0) { + dateType = ((List<?>) dateType).get(0); + meta.put(MetaConst.META_DATA_TYPE, dateType.toString()); + } + } else if (aliasObject instanceof SelectItemHelper.AliasFunExpr) { + SelectItemHelper.AliasFunExpr aliasDatetime = (SelectItemHelper.AliasFunExpr) aliasObject; + String dateType = aliasDatetime.getDateType(); + if (dateType != null) { + meta.put(MetaConst.META_DATA_TYPE, dateType); } - metaList.add(temp); } - return metaList; } class Converter { @@ -271,7 +350,7 @@ public class DruidDialect extends AbstractDataSourceDialect { * @return */ public Converter generateStandard() { - SQLQuerySource dbQuerySource = param.getDbQuerySource(); + SelectStatement dbQuerySource = sqlQueryContext.getDbSelectStatement(); if (StringUtil.isNotBlank(dbQuerySource.getLimit()) && dbQuerySource.getLimit().split(",").length >= 2) { @@ -284,9 +363,14 @@ public class DruidDialect extends AbstractDataSourceDialect { return this; } + @SneakyThrows + public Converter mergeFunctions() { + sql = FunctionsMergeHelper.build(sql); + return this; + } public Converter generateDateFunction() { - sql = SQLFunctionUtil.generateDateFunction(sql, DBTypeEnum.DRUID.getValue()); + sql = SQLFunctionUtil.generateDateFunction(sql, DBEngineType.DRUID.getValue()); return this; } @@ -307,19 +391,19 @@ public class DruidDialect extends AbstractDataSourceDialect { * @return */ public Optimizer generateQueryLimit() { - if (StringUtil.isEmpty(param.getEngineQuerySource())) { - setDefaultResultRows(param.getDbQuerySource(), engineConfigSource.getDefaultResultNum()); - setMaxCacheResultRows(param.getDbQuerySource(), engineConfigSource.getMaxCacheNum()); + if (StringUtil.isEmpty(sqlQueryContext.getFederationSelectStatement())) { + setDefaultResultRows(sqlQueryContext.getDbSelectStatement(), engineConfigSource.getDefaultResultNum()); + setMaxCacheResultRows(sqlQueryContext.getDbSelectStatement(), engineConfigSource.getMaxCacheNum()); } else { - setDefaultResultRows(param.getEngineQuerySource(), engineConfigSource.getDefaultResultNum()); - setMaxCacheResultRows(param.getDbQuerySource(), engineConfigSource.getMaxCacheNum()); - setMaxCacheResultRows(param.getEngineQuerySource(), engineConfigSource.getMaxCacheNum()); + setDefaultResultRows(sqlQueryContext.getFederationSelectStatement(), engineConfigSource.getDefaultResultNum()); + setMaxCacheResultRows(sqlQueryContext.getDbSelectStatement(), engineConfigSource.getMaxCacheNum()); + setMaxCacheResultRows(sqlQueryContext.getFederationSelectStatement(), engineConfigSource.getMaxCacheNum()); } return this; } - private void setDefaultResultRows(SQLQuerySource sqlQuerySource, int defaultResultRows) { + private void setDefaultResultRows(SelectStatement sqlQuerySource, int defaultResultRows) { if (StringUtil.isBlank(sqlQuerySource.getLimit())) { sqlQuerySource.setSqlBody(sqlQuerySource.getSqlBody() + " limit " + defaultResultRows); sqlQuerySource.setLimit(String.valueOf(defaultResultRows)); @@ -333,7 +417,7 @@ public class DruidDialect extends AbstractDataSourceDialect { * @param sqlQuerySource * @return */ - private void setMaxCacheResultRows(SQLQuerySource sqlQuerySource, int maxCacheResultRows) { + private void setMaxCacheResultRows(SelectStatement sqlQuerySource, int maxCacheResultRows) { if (!sqlQuerySource.isEnableLimit()) { return; } @@ -358,7 +442,7 @@ public class DruidDialect extends AbstractDataSourceDialect { } public Optimizer generateSQL() { - SQLQuerySource sqlQuerySource = param.getDbQuerySource(); + SelectStatement sqlQuerySource = sqlQueryContext.getDbSelectStatement(); sql = sqlQuerySource.getSqlBody(); return this; } @@ -374,8 +458,11 @@ public class DruidDialect extends AbstractDataSourceDialect { @Data class DruidQueryParam { private String query; - private Map<String, String> context = Maps.newHashMap(); + private Map<String, Object> context = Maps.newHashMap(); private String resultFormat; + private boolean header; + private boolean typesHeader; + private boolean sqlTypesHeader; } diff --git a/src/main/java/com/mesalab/qgw/dialect/FederationDialect.java b/src/main/java/com/mesalab/qgw/dialect/FederationDialect.java index e58fbd91..81c877f6 100644 --- a/src/main/java/com/mesalab/qgw/dialect/FederationDialect.java +++ b/src/main/java/com/mesalab/qgw/dialect/FederationDialect.java @@ -8,41 +8,41 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.jayway.jsonpath.JsonPath; -import com.mesalab.calcite.CalciteMemoryUtils; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.calcite.CalciteConnectionProcessor; +import com.mesalab.common.enums.DBEngineType; +import com.mesalab.common.utils.SpringContextUtil; +import com.mesalab.common.utils.sqlparser.*; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.DataTypeMapping; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.enums.QueryFormatEnum; -import com.mesalab.common.utils.sqlparser.SQLHelper; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.enums.OutputMode; +import com.mesalab.qgw.constant.MetaConst; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.EngineConfigSource; +import com.mesalab.qgw.model.basic.SQLQueryContext; +import com.mesalab.qgw.model.basic.SelectStatement; import com.mesalab.qgw.model.basic.udf.UDF; import com.geedgenetworks.utils.StringUtil; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.Statement; -import net.sf.jsqlparser.statement.select.*; import javax.annotation.Nullable; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * 联邦查询-基于Calcite支持多源联合查询 */ @Slf4j -public class FederationDialect extends AbstractEngineDialect{ +public class FederationDialect extends AbstractEngineDialect { private Stopwatch watch = Stopwatch.createUnstarted(); + private final EngineConfigSource engineConfigSource = (EngineConfigSource) SpringContextUtil.getBean("engineConfigSource"); + private static final Map<String, String> metaMap = Maps.newHashMap(); - public QueryProfile param; + public SQLQueryContext sqlQueryContext; public BaseResult baseResult; static { @@ -51,45 +51,45 @@ public class FederationDialect extends AbstractEngineDialect{ metaMap.put("VARCHAR", DataTypeMapping.STRING); } - public FederationDialect(QueryProfile param, BaseResult result) { - this.param = param; + public FederationDialect(SQLQueryContext sqlQueryContext, BaseResult result) { + this.sqlQueryContext = sqlQueryContext; this.baseResult = result; } @Override public void init() { - if (StringUtil.isNotEmpty(param.getEngineQuerySource())) { - param.getEngineQuerySource().setExecSQL(param.getEngineQuerySource().getSqlBody()); + if (StringUtil.isNotEmpty(sqlQueryContext.getFederationSelectStatement())) { + sqlQueryContext.getFederationSelectStatement().setExecSQL(sqlQueryContext.getFederationSelectStatement().getSqlBody()); } else { - SQLQuerySource sqlQuerySource = new SQLQuerySource(); - sqlQuerySource.setExecSQL(null); - param.setEngineQuerySource(sqlQuerySource); + SelectStatement fdSelectStatement = new SelectStatement(); + fdSelectStatement.setExecSQL(null); + sqlQueryContext.setFederationSelectStatement(fdSelectStatement); } } @Override public String getFederateQuery() { - return param.getEngineQuerySource().getExecSQL(); + return sqlQueryContext.getFederationSelectStatement().getExecSQL(); } @Override public String getSampleQuery() { - String sql = param.getEngineQuerySource().getExecSQL(); + String sql = sqlQueryContext.getFederationSelectStatement().getExecSQL(); if (!isExecute()) { return sql; } try { - sql = String.valueOf(convertSampleSql(sql)); + sql = new SampleSQLHelper(sql, null, false, engineConfigSource.getDefaultResultNum()).build(); } catch (JSQLParserException e) { log.error("syntax-check sql error {}, execute original sql: {}, error is: {} ", - sql, sql = param.getDbQuerySource().getSqlBody(), e.getMessage() == null ? e.getCause() : e.getMessage()); + sql, sql = sqlQueryContext.getDbSelectStatement().getSqlBody(), e.getMessage() == null ? e.getCause() : e.getMessage()); } return sql; } @Override public String getSyntaxCheckQuery() { - String sql = param.getEngineQuerySource().getExecSQL(); + String sql = sqlQueryContext.getFederationSelectStatement().getExecSQL(); if (!isExecute()) { return sql; } @@ -107,13 +107,13 @@ public class FederationDialect extends AbstractEngineDialect{ } @Override - final public BaseResult getProcesses(String queryId){ + final public BaseResult getProcesses(String queryId) { return null; } @Override public boolean isExecute() { - return StringUtil.isNotEmpty(param.getEngineQuerySource().getExecSQL()); + return StringUtil.isNotEmpty(sqlQueryContext.getFederationSelectStatement().getExecSQL()); } @@ -122,63 +122,73 @@ public class FederationDialect extends AbstractEngineDialect{ try { watch.start(); if (isExecute()) { - log.info("DB engine is :{},execute query is: {}", DBTypeEnum.ENGINE.getValue(), JSON.toJSONString(sql)); - SQLQuerySource sqlQuerySource = param.getEngineQuerySource(); - Map resultMap = CalciteMemoryUtils.executeMemoryQuery(sqlQuerySource.getTableNames().get(0), + log.info("DB engine is :{}, execute query is: {}", DBEngineType.QGW.getValue(), JSON.toJSONString(sql)); + SelectStatement fdSelectStatement = sqlQueryContext.getFederationSelectStatement(); + Map resultMap = CalciteConnectionProcessor.getInstance().executeMemoryQuery(fdSelectStatement.getTableNames().get(0), (List<Map<String, String>>) baseResult.getMeta(), (List<Map<String, Object>>) baseResult.getData(), sql); baseResult.setData(resultMap.get("data")); baseResult.setMeta(unifyMeta(resultMap.get("meta"))); } - log.debug("Calcite execute time :" + watch.elapsed().toMillis() / 1000 + "s"); - + long elapsed = watch.elapsed(TimeUnit.MILLISECONDS); + if (elapsed > engineConfigSource.getHighLatencyThreshold()) { + log.warn("DB engine is: {}, elapsed: {} ms, sql: {}", DBEngineType.QGW.getValue(), elapsed, sql); + } } catch (RuntimeException ex) { - log.error("Engine execute memory-query error:{}", ex); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - QGWErrorCode.SQL_QUERY_FEDERATION_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_QUERY_FEDERATION_EXCEPTION.getMessage(),ex.getMessage())); + log.error("Federation query error:{}", ex.getMessage()); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.SQL_FEDERATION_QUERY_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_FEDERATION_QUERY_EXCEPTION.getMessage(), ex.getMessage())); } return build(); } - private BaseResult build() { - - long executeTime = watch.elapsed(TimeUnit.MILLISECONDS); Map<String, Object> statisticMap = (Map<String, Object>) baseResult.getStatistics(); - statisticMap.put("elapsed", Long.parseLong(String.valueOf(statisticMap.get("elapsed"))) + executeTime); if (StringUtil.isEmpty(baseResult.getData())) { statisticMap.put("result_bytes", 0); statisticMap.put("result_rows", 0); } else { statisticMap.put("result_bytes", JSON.toJSONBytes(baseResult.getData(), JSONWriter.Feature.LargeObject).length); - statisticMap.put("result_rows", ((List) baseResult.getData()).size()); + statisticMap.put("result_rows", ((List<?>) baseResult.getData()).size()); } - extendMetaInfo(baseResult.getMeta()); - if (param.getFormat().equalsIgnoreCase(QueryFormatEnum.CSV.getValue())) { - convertJsonToCSV(); + baseResult.setMeta(ColumnCategoryHelper.expandMetaCategory(baseResult.getMeta(), sqlQueryContext.getFederationSelectStatement(), null)); + if (baseResult.getMeta() != null + && baseResult.getData() != null + && ((List<Map<String, Object>>) baseResult.getMeta()).stream().anyMatch(x -> DataTypeMapping.LONG.equalsIgnoreCase(String.valueOf(x.get("type"))))) { + + List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); + for (int i = 0; i < data.size(); i++) { + Map<String, Object> map = data.get(i); + String jsonString = JSON.toJSONString(map, JSONWriter.Feature.WriteNulls, JSONWriter.Feature.BrowserCompatible); + Map parsedMap = JSON.parseObject(jsonString, LinkedHashMap.class); + map.clear(); + map.putAll(parsedMap); + } + baseResult.setData(data); } - baseResult.setFormatType(param.getFormat()); - log.debug("Execute build Time : {}", watch.elapsed(TimeUnit.MILLISECONDS) / 1000.0); - return baseResult; - } + if (baseResult.getMeta() != null) { + List<Map<String, Object>> metaList = (List<Map<String, Object>>) baseResult.getMeta(); + metaList.forEach(x -> { + x.remove(MetaConst.META_CATEGORY); + x.remove(MetaConst.META_DATA_TYPE); + x.remove(MetaConst.META_FIELD_NAME); + }); - private void extendMetaInfo(Object meta) { - if (StringUtil.isEmpty(meta)) { - return; } - List<Map<String, Object>> data = JsonPath.read(meta, "$"); - SQLQuerySource querySource = StringUtil.isEmpty(param.getEngineQuerySource()) || StringUtil.isEmpty(param.getEngineQuerySource().getExecSQL()) ? param.getDbQuerySource() : param.getEngineQuerySource(); - Map<String, String> groupDimension = querySource.getGroupDimension(); - for (Map<String, Object> map : data) { - String field = String.valueOf(map.get("name")); - if (StringUtil.isNotEmpty(groupDimension) && ((groupDimension.containsKey(field) || groupDimension.containsValue(field)))) { - map.put("category", "Dimension"); - } else { - map.put("category", "Metric"); - } + if (sqlQueryContext.getFormat().equalsIgnoreCase(OutputMode.CSV.getValue())) { + convertJsonToCSV(); + } else if (sqlQueryContext.getFormat().equalsIgnoreCase(OutputMode.JSON.getValue())) { + baseResult.setMeta(null); + } else if (sqlQueryContext.getFormat().equalsIgnoreCase(OutputMode.JSON_COMPACT.getValue())) { + convertToJsonCompact(); } + long executeTime = watch.elapsed(TimeUnit.MILLISECONDS); + statisticMap.put("elapsed", Long.parseLong(String.valueOf(statisticMap.get("elapsed"))) + executeTime); + baseResult.setOutputMode(sqlQueryContext.getFormat()); + log.debug("Federation execute and build result time : {} ms", executeTime); + return baseResult; } private List<Map<String, String>> unifyMeta(Object meta) { @@ -196,12 +206,11 @@ public class FederationDialect extends AbstractEngineDialect{ } @Override - public void executeUDF() { - SQLQuerySource sqlQuerySource = param.getDbQuerySource(); + public void executeUDF(boolean isDryRun) { + SelectStatement sqlQuerySource = sqlQueryContext.getDbSelectStatement(); Set<UDF> udfSet = sqlQuerySource.getUdfSet(); for (UDF udf : udfSet) { - List<Map<String, String>> targetResult = (List<Map<String, String>>) udf.execute(sqlQuerySource, - baseResult); + List<Map<String, String>> targetResult = (List<Map<String, String>>) udf.execute(sqlQuerySource, isDryRun, baseResult); baseResult.setData(targetResult); } } @@ -209,7 +218,7 @@ public class FederationDialect extends AbstractEngineDialect{ private void convertJsonToCSV() { - List<String> results = Lists.transform( (List<Map<String, Object>>) baseResult.getData(), new Function<Map<String, Object>, String>() { + List<String> results = Lists.transform((List<Map<String, Object>>) baseResult.getData(), new Function<Map<String, Object>, String>() { @Override public String apply(@Nullable Map<String, Object> input) { @@ -229,55 +238,12 @@ public class FederationDialect extends AbstractEngineDialect{ baseResult.setMeta(Joiner.on(",").useForNull("").join(metas)); } - /** - * Desc: where、having至无效：添加 OR 1=1 - * - * @param sql - * @return {@link SelectBody} - * @created by wWei - * @date 2022/4/27 5:18 下午 - */ - private SelectBody convertSampleSql(String sql) throws JSQLParserException { - Statement parse = CCJSqlParserUtil.parse(sql); - if (parse instanceof Select) { - Select select = (Select) parse; - SelectBody selectBody = select.getSelectBody(); - if (selectBody instanceof PlainSelect) { - PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); - FromItem fromItem = plainSelect.getFromItem(); - if (fromItem instanceof SubSelect) { - SubSelect subSelect = (SubSelect) plainSelect.getFromItem(); - subSelect.setSelectBody(convertSampleSql(subSelect.getSelectBody().toString())); - } - if (StringUtil.isNotEmpty(plainSelect.getWhere())) { - plainSelect.setWhere(generateDatasetWhereSql(plainSelect.getWhere().toString())); - } - if (StringUtil.isNotEmpty(plainSelect.getHaving())) { - plainSelect.setHaving(generateDatasetWhereSql(plainSelect.getHaving().toString())); - } - return plainSelect; - } else if (selectBody instanceof SetOperationList) { - SetOperationList setOperationList = (SetOperationList) selectBody; - List<SelectBody> selects = setOperationList.getSelects(); - for (int i = 0; i < selects.size(); i++) { - selects.set(i, convertSampleSql(selects.get(i).toString())); - } - return setOperationList; - } else { - throw new QGWBusinessException(ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.SQL_PARSE_ONLY_SUPPORT_SELECT)); - } - } - throw new QGWBusinessException(ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.SQL_PARSE_ONLY_SUPPORT_SELECT)); - } - - private Expression generateDatasetWhereSql(String where) throws JSQLParserException { - String sql = String.format("SELECT * FROM tableName WHERE %s %s ", where, "OR 1=1"); - Statement parse = CCJSqlParserUtil.parse(sql); - Select select = (Select) parse; - PlainSelect selectBody = (PlainSelect) select.getSelectBody(); - return selectBody.getWhere(); + private void convertToJsonCompact() { + List<Map<String, Object>> dataList = (List<Map<String, Object>>) baseResult.getData(); + List<Object> result = dataList.stream() + .map(Map::values) + .collect(Collectors.toList()); + baseResult.setData(result); } } diff --git a/src/main/java/com/mesalab/qgw/dialect/HbaseDialect.java b/src/main/java/com/mesalab/qgw/dialect/HbaseDialect.java index f8898663..d40fc74c 100644 --- a/src/main/java/com/mesalab/qgw/dialect/HbaseDialect.java +++ b/src/main/java/com/mesalab/qgw/dialect/HbaseDialect.java @@ -7,20 +7,21 @@ import com.google.common.collect.Maps; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; import com.mesalab.common.entity.DataTypeMapping; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.common.enums.QueryFormatEnum; +import com.mesalab.common.enums.DBEngineType; +import com.mesalab.common.enums.OutputMode; import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.utils.SpringContextUtil; +import com.mesalab.common.utils.sqlparser.ColumnCategoryHelper; import com.mesalab.common.utils.sqlparser.SQLVisitorUtil; import com.mesalab.common.utils.sqlparser.SQLFunctionUtil; import com.mesalab.qgw.constant.QGWMessageConst; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.qgw.model.basic.QueryProfile; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.SQLQueryContext; import com.mesalab.qgw.model.basic.EngineConfigSource; import com.mesalab.qgw.model.basic.HBaseAPISource; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.qgw.model.basic.SelectStatement; import com.geedgenetworks.utils.StringUtil; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; @@ -40,7 +41,6 @@ import java.util.concurrent.TimeUnit; @Slf4j public class HbaseDialect extends AbstractDataSourceDialect { - public QueryProfile param; public BaseResult baseResult; private Stopwatch watch = Stopwatch.createUnstarted(); private static final String URL_PREFIX = "jdbc:phoenix:"; @@ -49,6 +49,10 @@ public class HbaseDialect extends AbstractDataSourceDialect { private final static Map<String, String> metaMap = Maps.newHashMap(); public static Properties PROPERTIES = new Properties(); + public HbaseDialect(SQLQueryContext sqlQueryContext) { + super(sqlQueryContext); + } + static { try { Class.forName("org.apache.phoenix.jdbc.PhoenixDriver"); @@ -75,55 +79,47 @@ public class HbaseDialect extends AbstractDataSourceDialect { metaMap.put("VARCHAR", DataTypeMapping.STRING); } - public HbaseDialect(QueryProfile param) { - this.param = param; - } @Override void init() { - if (StringUtil.isNotEmpty(param.getDbQuerySource())) { - param.getDbQuerySource().setExecSQL(param.getDbQuerySource().getSqlBody()); + if (StringUtil.isNotEmpty(sqlQueryContext.getDbSelectStatement())) { + sqlQueryContext.getDbSelectStatement().setExecSQL(sqlQueryContext.getDbSelectStatement().getSqlBody()); } else { - SQLQuerySource sqlQuerySource = new SQLQuerySource(); - sqlQuerySource.setExecSQL(param.getQuery()); - param.setDbQuerySource(sqlQuerySource); + SelectStatement dbSelectStatement = new SelectStatement(); + dbSelectStatement.setExecSQL(sqlQueryContext.getOriginalSQL()); + sqlQueryContext.setDbSelectStatement(dbSelectStatement); } baseResult = BaseResultGenerator.success(); } @Override String convertQuery(String sql) { - sql = SQLFunctionUtil.generateDateFunction(sql, DBTypeEnum.HBASE.getValue()); + sql = SQLFunctionUtil.generateDateFunction(sql, DBEngineType.HBASE.getValue()); try { Select select = (Select) CCJSqlParserUtil.parse(sql); select.getSelectBody().accept(SQLVisitorUtil.getVisitorOfEscapeMetadataWithDoubleQuote()); return select.toString(); } catch (JSQLParserException | RuntimeException e) { log.error("SQL Syntax Error: Converter SQL Syntax Error, SQL is:{}, Error is:{}", sql, e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); } } @Override - String getBestQuery() { + String generateRuleBaseOptimizedQuery() { return new Optimizer().generateQueryLimit().generateSQL().build(); } @Override - String getOriginalQuery() { - return param.getQuery(); - } - - @Override String getSampleQuery() { - return convertQuery(param.getQuery()); + return convertQuery(sqlQueryContext.getOriginalSQL()); } @Override String getExplainQuery() { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), QGWMessageConst.HBASE_DIALECT_UNSUPPORTED_EXPLAIN_SYNTAX)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), QGWMessageConst.HBASE_DIALECT_UNSUPPORTED_EXPLAIN_SYNTAX)); } @Override @@ -136,17 +132,28 @@ public class HbaseDialect extends AbstractDataSourceDialect { } @Override + List<Map<String, String>> buildExampleMeta() { + return baseResult.getMeta() != null ? (List<Map<String, String>>) baseResult.getMeta() : null; + } + + @Override + List<Object> buildExampleData() { + return baseResult.getData() != null ? (List<Object>) baseResult.getData() : null; + } + + @Override public BaseResult generateBaseResult(String sql, Optional<String> message) { watch.start(); Map<String, Object> results = executeQuery(sql); int status = Integer.parseInt(String.valueOf(results.get("status"))); - if (status == ResultStatusEnum.SUCCESS.getCode()) { - baseResult = BaseResultGenerator.generate(status, ResultCodeEnum.SUCCESS.getCode(), message.get(), - results.get("data"), results.get("meta"), (Map<String, Object>) results.get("statistics"), QueryFormatEnum.JSON.getValue()); + if (status == HttpStatusCodeEnum.SUCCESS.getCode()) { + baseResult = BaseResultGenerator.generate(status, ResultCodeEnum.SUCCESS.getCode(), true, message.get(), + results.get("data"), results.get("meta"), (Map<String, Object>) results.get("statistics"), OutputMode.JSON.getValue()); } else { - baseResult = BaseResultGenerator.generate(status, QGWErrorCode.SQL_BUILDER_EXCEPTION.getCode(), String.valueOf(results.get("message")), - results.get("data"), results.get("meta"), (Map<String, Object>) results.get("statistics"), param.getFormat()); + baseResult = BaseResultGenerator.generate(status, CommonErrorCode.SQL_REWRITE_AND_TRANSFORMATION_EXCEPTION.getCode(), false, String.valueOf(results.get("message")), + results.get("data"), results.get("meta"), (Map<String, Object>) results.get("statistics"), sqlQueryContext.getFormat()); } + baseResult.setMeta(ColumnCategoryHelper.expandMetaCategory(baseResult.getMeta(), sqlQueryContext.getDbSelectStatement(), sqlQueryContext.getDbEngine())); return baseResult; } @@ -155,7 +162,7 @@ public class HbaseDialect extends AbstractDataSourceDialect { java.sql.Statement statement = null; ResultSet resultSet = null; try { - log.info("{} engine execute query: {}", DBTypeEnum.HBASE.getValue(), sql); + log.info("{} engine execute query: {}", DBEngineType.HBASE.getValue(), sql); PROPERTIES.setProperty("hbase.rpc.timeout", hBaseAPISource.getRpcTimeout()); PROPERTIES.setProperty("hbase.client.scanner.timeout.period", hBaseAPISource.getRpcTimeout()); PROPERTIES.setProperty("phoenix.query.timeoutMs", hBaseAPISource.getRpcTimeout()); @@ -165,9 +172,9 @@ public class HbaseDialect extends AbstractDataSourceDialect { resultSet = statement.executeQuery(sql); return buildResult(resultSet); } catch (RuntimeException | SQLException e) { - log.error("{} engine error: {}", DBTypeEnum.HBASE.getValue(), e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),simplyErrorMessage(e.getMessage()))); + log.error("{} engine error: {}", DBEngineType.HBASE.getValue(), e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), simplyErrorMessage(e.getMessage()))); } finally { close(resultSet, statement, conn); } @@ -197,7 +204,7 @@ public class HbaseDialect extends AbstractDataSourceDialect { statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); statistics.put("result_bytes", dataList.toString().getBytes().length); statistics.put("result_rows", dataList.size()); - result.put("status", ResultStatusEnum.SUCCESS.getCode()); + result.put("status", HttpStatusCodeEnum.SUCCESS.getCode()); result.put("statistics", statistics); result.put("meta", metaDataList); result.put("data", dataList); @@ -211,12 +218,12 @@ public class HbaseDialect extends AbstractDataSourceDialect { @Override public BaseResult executeKillQuery(String queryId) { - return BaseResultGenerator.failure(ResultCodeEnum.PARAMETER_ERROR.getCode(), String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"not support")); + return BaseResultGenerator.failure(CommonErrorCode.PARAMETER_ERROR.getCode(), String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), "not support")); } @Override public BaseResult getProcesses(String queryId) { - return BaseResultGenerator.failure(ResultCodeEnum.PARAMETER_ERROR.getCode(), String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"not support")); + return BaseResultGenerator.failure(CommonErrorCode.PARAMETER_ERROR.getCode(), String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), "not support")); } class Optimizer { @@ -229,11 +236,11 @@ public class HbaseDialect extends AbstractDataSourceDialect { */ public Optimizer generateQueryLimit() { - setDefaultResultRows(StringUtil.isEmpty(param.getEngineQuerySource()) ? param.getDbQuerySource() : param.getEngineQuerySource(), engineConfigSource.getDefaultResultNum()); + setDefaultResultRows(StringUtil.isEmpty(sqlQueryContext.getFederationSelectStatement()) ? sqlQueryContext.getDbSelectStatement() : sqlQueryContext.getFederationSelectStatement(), engineConfigSource.getDefaultResultNum()); return this; } - private void setDefaultResultRows(SQLQuerySource sqlQuerySource, int defaultResultRows) { + private void setDefaultResultRows(SelectStatement sqlQuerySource, int defaultResultRows) { if (StringUtil.isBlank(sqlQuerySource.getLimit())) { sqlQuerySource.setSqlBody(sqlQuerySource.getSqlBody() + " limit " + defaultResultRows); sqlQuerySource.setLimit(String.valueOf(defaultResultRows)); @@ -242,7 +249,7 @@ public class HbaseDialect extends AbstractDataSourceDialect { } public Optimizer generateSQL() { - SQLQuerySource sqlQuerySource = param.getDbQuerySource(); + SelectStatement sqlQuerySource = sqlQueryContext.getDbSelectStatement(); sql = sqlQuerySource.getSqlBody(); return this; } diff --git a/src/main/java/com/mesalab/qgw/exception/QGWBusinessException.java b/src/main/java/com/mesalab/qgw/exception/QGWBusinessException.java index b4932405..bbec18d8 100644 --- a/src/main/java/com/mesalab/qgw/exception/QGWBusinessException.java +++ b/src/main/java/com/mesalab/qgw/exception/QGWBusinessException.java @@ -1,7 +1,7 @@ package com.mesalab.qgw.exception; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.CommonErrorCode; import lombok.*; /** @@ -20,12 +20,12 @@ public class QGWBusinessException extends RuntimeException { /** * 状态码 */ - private int status = ResultStatusEnum.SERVER_ERROR.getCode(); + private int status = HttpStatusCodeEnum.SERVER_ERROR.getCode(); /** * 业务码 */ - private String code = ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(); + private String code = CommonErrorCode.UNKNOWN_EXCEPTION.getCode(); /** @@ -33,14 +33,14 @@ public class QGWBusinessException extends RuntimeException { */ private String message; - public QGWBusinessException(int status, QGWErrorCode errorCode) { + public QGWBusinessException(int status, CommonErrorCode errorCode) { super(errorCode.getMessage()); this.status = status; this.code = errorCode.getCode(); this.message = errorCode.getMessage(); } - public QGWBusinessException(QGWErrorCode errorCode) { + public QGWBusinessException(CommonErrorCode errorCode) { super(errorCode.getMessage()); this.code = errorCode.getCode(); this.message = errorCode.getMessage(); diff --git a/src/main/java/com/mesalab/qgw/exception/QGWErrorCode.java b/src/main/java/com/mesalab/qgw/exception/QGWErrorCode.java deleted file mode 100644 index 1024fa9a..00000000 --- a/src/main/java/com/mesalab/qgw/exception/QGWErrorCode.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.mesalab.qgw.exception; - -import lombok.Getter; - -/** - * @Author wxs - * @Date 2022/11/3 - */ -@Getter -public enum QGWErrorCode { - - /** - * 1-3位：异常类型（HTTP协议状态码） - * 3-4位：模块 01 - * 5-7：自然排序 - */ - // SQL解析错误 - SQL_SYNTAX_PARSE_EXCEPTION("40001300", "Syntax error in SQL statement: %s"), - // 数据库引擎错误 - SQL_EXECUTION_BAD_REQUEST_EXCEPTION("40001301", "Invalid SQL statement error in database execution engine: %s"), - //Schema与DB不一致 - SCHEMA_WITCH_DB_INCONSISTENTY("50001010", "Schema inconsistent with DB: %s"), - // SQL构建错误 - SQL_BUILDER_EXCEPTION("50001100", "Error in SQL query builder and optimizer: %s"), - // SQL在数据库查询中执行错误 - SQL_EXECUTION_SERVER_EXCEPTION("50001300", "Error in database execution engine: %s"), - // SQL在联邦查询中执行错误 - SQL_QUERY_FEDERATION_EXCEPTION("50001500", "Error in query federation: %s"), - ; - private String code; - private String message; - - QGWErrorCode(String code, String message) { - this.code = code; - this.message = message; - } -} diff --git a/src/main/java/com/mesalab/qgw/filter/ControllerFilter.java b/src/main/java/com/mesalab/qgw/filter/ControllerFilter.java index dc981837..4646b027 100644 --- a/src/main/java/com/mesalab/qgw/filter/ControllerFilter.java +++ b/src/main/java/com/mesalab/qgw/filter/ControllerFilter.java @@ -6,16 +6,12 @@ import javax.servlet.*; import javax.servlet.annotation.WebFilter; import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; @Component @WebFilter(urlPatterns = "/*") -public class ControllerFilter implements Filter { - - private static final Set<String> NOT_FILTER_PATHS = Collections.unmodifiableSet(new HashSet<>( - Collections.singletonList("/knowledge_base/v1"))); +public class ControllerFilter implements Filter { + private static final String NOT_FILTER_PATHS_KB = "/v1/knowledge_base"; + private static final String NOT_FILTER_PATHS_HOS = "/v1/hos"; @Override public void init(FilterConfig filterConfig) throws ServletException { @@ -26,7 +22,7 @@ public class ControllerFilter implements Filter { public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { HttpServletRequest httpServletRequest = ((HttpServletRequest) servletRequest); String servletPath = httpServletRequest.getServletPath(); - if (NOT_FILTER_PATHS.contains(servletPath)) { + if (String.valueOf(servletPath).startsWith(NOT_FILTER_PATHS_HOS) || String.valueOf(servletPath).startsWith(NOT_FILTER_PATHS_KB)) { filterChain.doFilter(servletRequest, servletResponse); } else { filterChain.doFilter(new RequestParamWrapper(httpServletRequest), servletResponse); diff --git a/src/main/java/com/mesalab/qgw/filter/RequestParamWrapper.java b/src/main/java/com/mesalab/qgw/filter/RequestParamWrapper.java index ea1c6cc6..82780646 100644 --- a/src/main/java/com/mesalab/qgw/filter/RequestParamWrapper.java +++ b/src/main/java/com/mesalab/qgw/filter/RequestParamWrapper.java @@ -3,7 +3,7 @@ package com.mesalab.qgw.filter; import com.google.common.collect.Lists; -import com.mesalab.common.enums.QueryParamEnum; +import com.mesalab.common.enums.QueryParam; import com.mesalab.common.utils.HttpHelper; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; @@ -34,12 +34,12 @@ public class RequestParamWrapper extends HttpServletRequestWrapper { @Override public String[] getParameterValues(String name) { - if (QueryParamEnum.QUERY.getValue().equals(name)) { + if (QueryParam.QUERY.getValue().equals(name)) { queryString = queryString.replaceAll("\\+", "%2B"); - List<NameValuePair> values = URLEncodedUtils.parse(queryString, Charset.forName("UTF-8")); + List<NameValuePair> values = URLEncodedUtils.parse(queryString, StandardCharsets.UTF_8); List<String> valueList = Lists.newArrayList(); for (NameValuePair nameValuePair: values) { - if (nameValuePair.getName().equals(QueryParamEnum.QUERY.getValue())) { + if (nameValuePair.getName().equals(QueryParam.QUERY.getValue())) { valueList.add(nameValuePair.getValue()); } } diff --git a/src/main/java/com/mesalab/qgw/interceptor/GlobalExceptionHandler.java b/src/main/java/com/mesalab/qgw/interceptor/GlobalExceptionHandler.java index e30f24fb..111e4b19 100644 --- a/src/main/java/com/mesalab/qgw/interceptor/GlobalExceptionHandler.java +++ b/src/main/java/com/mesalab/qgw/interceptor/GlobalExceptionHandler.java @@ -1,18 +1,22 @@ package com.mesalab.qgw.interceptor; import cn.hutool.core.exceptions.ExceptionUtil; +import cn.hutool.core.util.StrUtil; import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException;; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.utils.MDCUtil; +import com.mesalab.common.utils.RandomUtil; import com.mesalab.qgw.exception.QGWBusinessException; import com.geedgenetworks.utils.StringUtil; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.context.request.async.AsyncRequestTimeoutException; + import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -23,6 +27,7 @@ public class GlobalExceptionHandler { /** * 捕获特定异常 + * * @param e * @param request */ @@ -34,29 +39,28 @@ public class GlobalExceptionHandler { @ExceptionHandler(Exception.class) public BaseResult handleException(Exception e, HttpServletRequest request, HttpServletResponse response) { - response.setStatus(ResultStatusEnum.BAD_REQUEST.getCode()); - String queryKey = DigestUtil.md5Hex(StringUtil.createUUID()); - log.error("queryKey:{}, message:{}, stackTrace:{}", queryKey, ExceptionUtil.getRootCauseMessage(e), getStackTrace(e)); + response.setStatus(HttpStatusCodeEnum.BAD_REQUEST.getCode()); + String queryKey = StrUtil.isEmpty(MDCUtil.getTraceId()) ? RandomUtil.getUUID() : MDCUtil.getTraceId(); + log.error("requestId:{}, message:{}, stackTrace:{}", queryKey, ExceptionUtil.getRootCauseMessage(e), getStackTrace(e)); return BaseResultGenerator.failure(e.getMessage(), queryKey); } - @ExceptionHandler({QGWBusinessException.class}) public BaseResult handleBusinessException(QGWBusinessException e, HttpServletRequest request, HttpServletResponse response) { response.setStatus(e.getStatus()); - String queryKey = DigestUtil.md5Hex(StringUtil.createUUID()); + String queryKey = StrUtil.isEmpty(MDCUtil.getTraceId()) ? RandomUtil.getUUID() : MDCUtil.getTraceId(); String message = (null != e.getMessage()) ? e.getMessage() : ExceptionUtil.getRootCauseMessage(e); - log.error("queryKey:{}.message:{}.stackTrace:{}", queryKey, message, getStackTrace(e)); - return BaseResultGenerator.failure(e.getStatus(), e.getCode(), queryKey, message); + log.error("requestId:{}.message:{}.stackTrace:{}", queryKey, message, getStackTrace(e)); + return BaseResultGenerator.failure(e.getStatus(), e.getCode(), queryKey, message); } @ExceptionHandler({BusinessException.class}) public BaseResult handleBusinessException(BusinessException e, HttpServletRequest request, HttpServletResponse response) { response.setStatus(e.getStatus()); - String queryKey = DigestUtil.md5Hex(StringUtil.createUUID()); + String queryKey = StrUtil.isEmpty(MDCUtil.getTraceId()) ? RandomUtil.getUUID() : MDCUtil.getTraceId(); String message = (null != e.getMessage()) ? e.getMessage() : ExceptionUtil.getRootCauseMessage(e); - log.error("queryKey:{}.message:{}.stackTrace:{}", queryKey, message, getStackTrace(e)); + log.error("requestId:{}.message:{}.stackTrace:{}", queryKey, message, getStackTrace(e)); return BaseResultGenerator.failure(e.getStatus(), e.getCode(), queryKey, message); } @@ -64,6 +68,4 @@ public class GlobalExceptionHandler { return StringUtil.isNotEmpty(e.getStackTrace()) ? e.getStackTrace()[0].toString() : ""; } - - - } +} diff --git a/src/main/java/com/mesalab/qgw/interceptor/QuerySubmitInterceptor.java b/src/main/java/com/mesalab/qgw/interceptor/QuerySubmitInterceptor.java index 1f5dcfac..97a5575b 100644 --- a/src/main/java/com/mesalab/qgw/interceptor/QuerySubmitInterceptor.java +++ b/src/main/java/com/mesalab/qgw/interceptor/QuerySubmitInterceptor.java @@ -7,10 +7,12 @@ import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONWriter; import com.google.common.cache.CacheStats; import com.mesalab.common.utils.HttpHelper; +import com.mesalab.common.utils.MDCUtil; import com.mesalab.common.utils.QueryCacheUtils; import com.mesalab.qgw.model.basic.CachedSubmitCheck; import com.geedgenetworks.utils.StringUtil; import lombok.Data; +import org.slf4j.MDC; import org.springframework.stereotype.Component; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.method.HandlerMethod; @@ -24,9 +26,12 @@ public class QuerySubmitInterceptor extends HandlerInterceptorAdapter { private static final Log log = LogFactory.get(); @Override public boolean preHandle(HttpServletRequest request, - HttpServletResponse response, Object handler) throws Exception - { - + HttpServletResponse response, Object handler) throws Exception { + String traceId = request.getHeader("Request-Id"); + if (traceId == null || traceId.isEmpty()) { + traceId = MDCUtil.generateTraceId(); + } + MDC.put(MDCUtil.TRACE_ID, traceId); if (handler instanceof HandlerMethod) { HandlerMethod handlerMethod = (HandlerMethod) handler; CachedSubmitCheck submitCheck = handlerMethod.getMethodAnnotation(CachedSubmitCheck.class); diff --git a/src/main/java/com/mesalab/qgw/listener/KnowledgeListener.java b/src/main/java/com/mesalab/qgw/listener/KnowledgeListener.java deleted file mode 100644 index af68b7da..00000000 --- a/src/main/java/com/mesalab/qgw/listener/KnowledgeListener.java +++ /dev/null @@ -1,185 +0,0 @@ -package com.mesalab.qgw.listener; - -import cn.hutool.core.io.IoUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.alibaba.fastjson2.JSON; -import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.config.listener.AbstractListener; -import com.alibaba.nacos.api.exception.NacosException; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.mesalab.common.nacos.NacosConst; -import com.mesalab.common.utils.IPUtil; -import com.mesalab.qgw.model.basic.HttpConfig; -import com.mesalab.qgw.service.impl.HttpClientService; -import com.mesalab.services.common.entity.KnowledgeConfig; -import com.mesalab.services.common.entity.KnowledgeBase; -import com.mesalab.services.common.entity.KnowledgeConstant; -import com.mesalab.services.configuration.HosConfig; -import com.geedgenetworks.utils.StringUtil; -import org.apache.http.Header; -import org.apache.http.message.BasicHeader; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @Author wxs - * @Date 2022/7/7 - */ -@Component -public class KnowledgeListener { - - private static final Log log = LogFactory.get(); - - - @Autowired - private HttpConfig httpConfig; - @Autowired - KnowledgeConfig knowledgeConfig; - @Autowired - private HttpClientService httpClientService; - @Autowired - private HosConfig hosConfig; - - @Autowired - private ConfigService pubConfigService; - - @Autowired - private ConfigService systemConfigService; - - @Value("${nacos.config.group}") - private String groupId; - - @Value("${nacos.config.data-id}") - private String dataId; - private static Map<String, String> updateMap = new HashMap<>(); - - @PostConstruct - private void initKnowledge() { - try { - loadIpLocation(pubConfigService.getConfig(NacosConst.KNOWLEDGE_BASE_DATA_ID, NacosConst.DEFAULT_GROUP, httpConfig.getServerRequestTimeOut())); - } catch (RuntimeException | NacosException e) { - log.error("loadIpLocation error: {}", e.getMessage()); - } - } - - @PostConstruct - private void registerKnowledgeBaseListener() { - try { - pubConfigService.addListener(NacosConst.KNOWLEDGE_BASE_DATA_ID, NacosConst.DEFAULT_GROUP, new AbstractListener() { - @Override - public void receiveConfigInfo(String content) { - loadIpLocation(content); - } - }); - } catch (RuntimeException | NacosException e) { - log.error("register knowledgeBase listener error: {}", e.getMessage()); - } - } - - @PostConstruct - private void registerSystemConfigListener() { - try { - systemConfigService.addListener(dataId, groupId, new AbstractListener() { - @Override - public void receiveConfigInfo(String versionInfo) { - try { - loadIpLocation(pubConfigService.getConfig(NacosConst.KNOWLEDGE_BASE_DATA_ID, NacosConst.DEFAULT_GROUP, httpConfig.getServerRequestTimeOut())); - } catch (NacosException e) { - log.error("loadIpLocation error: {}", e.getMessage()); - } - } - }); - } catch (RuntimeException | NacosException e) { - log.error("register systemConfig listener error: {}", e.getMessage()); - } - } - - private void loadIpLocation(String knowledgeMetadata) { - if (isLoadIpLocation(knowledgeMetadata)) { - IPUtil.ipLookupBuild(); - log.info("ip location build success"); - } else { - log.info("ip location no change"); - } - } - - private boolean isLoadIpLocation(String knowledgeMetadata) { - Map<String, KnowledgeBase> map = buildIPKnowledgeInfo(getKnowledgeBases(knowledgeMetadata)); - boolean isUpdate = false; - for (Map.Entry<String, KnowledgeBase> entry : map.entrySet()) { - KnowledgeBase knowledgeBase = entry.getValue(); - if (StringUtil.equals(knowledgeBase.getSha256(), updateMap.get(knowledgeBase.getId()))) { - continue; - } - log.info("{}.{} version: {}, knowledge updated.", knowledgeBase.getName(), knowledgeBase.getFormat(), knowledgeBase.getVersion()); - download(knowledgeBase); - updateMap.put(knowledgeBase.getId(), knowledgeBase.getSha256()); - isUpdate = true; - } - return isUpdate; - } - - private void download(KnowledgeBase knowledgeBase) { - FileOutputStream outputStream = null; - InputStream inputStream = null; - try { - Header header = new BasicHeader(KnowledgeConstant.TOKEN, hosConfig.getToken()); - inputStream = httpClientService.httpGetInputStream(knowledgeBase.getPath(), httpConfig.getServerResponseTimeOut(), header); - if (inputStream == null) { - log.error("download file error, skip this update, knowledge is: {}", knowledgeBase); - return; - } - outputStream = new FileOutputStream(KnowledgeConstant.DAT.concat(File.separator).concat(knowledgeBase.getName()).concat(".").concat(knowledgeBase.getFormat())); - IoUtil.copy(inputStream, outputStream); - log.info("knowledge download name: {}, version: {}", knowledgeBase.getName(), knowledgeBase.getVersion()); - } catch (IOException e) { - log.error("download file error, message: {}", e.getMessage()); - } finally { - IoUtil.close(inputStream); - IoUtil.close(outputStream); - } - } - - private List<KnowledgeBase> getKnowledgeBases(String knowledgeMetadata) { - log.info("knowledge base meta is: {}", knowledgeMetadata); - if (StringUtil.isEmpty(knowledgeMetadata)) { - return Lists.newArrayList(); - } - return JSON.parseArray(knowledgeMetadata, KnowledgeBase.class); - } - - private Map<String, KnowledgeBase> buildIPKnowledgeInfo(List<KnowledgeBase> knowledgeBaseList) { - Map<String, KnowledgeBase> map = Maps.newHashMap(); - String ipBuiltIn = knowledgeConfig.getIpBuiltIn(); - String ipUserDefined = knowledgeConfig.getIpUserDefined(); - String ipAsn = knowledgeConfig.getIpAsn(); - for (KnowledgeBase knowledgeBase : knowledgeBaseList) { - String name = knowledgeBase.getName(); - String version = knowledgeBase.getVersion(); - String concat = name.concat(":").concat(version); - if (StringUtil.equals(concat, ipBuiltIn) - || (StringUtil.equals(name, ipBuiltIn) && StringUtil.equals(version, KnowledgeConstant.LATEST))) { - map.put(KnowledgeConstant.IP_BUILTIN, knowledgeBase); - } else if (StringUtil.equals(concat, ipUserDefined) - || (StringUtil.equals(name, ipUserDefined) && StringUtil.equals(version, KnowledgeConstant.LATEST))) { - map.put(KnowledgeConstant.IP_USER_DEFINED, knowledgeBase); - } else if (StringUtil.equals(concat, ipAsn) - || (StringUtil.equals(name, ipUserDefined) && StringUtil.equals(version, KnowledgeConstant.LATEST))) { - map.put(KnowledgeConstant.ASN_BUILTIN, knowledgeBase); - } - } - return map; - } -} diff --git a/src/main/java/com/mesalab/qgw/listener/KnowledgeScheduler.java b/src/main/java/com/mesalab/qgw/listener/KnowledgeScheduler.java new file mode 100644 index 00000000..38c3d8fe --- /dev/null +++ b/src/main/java/com/mesalab/qgw/listener/KnowledgeScheduler.java @@ -0,0 +1,117 @@ +package com.mesalab.qgw.listener; + +import cn.hutool.core.io.IoUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.google.common.collect.Maps; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.utils.IPUtil; +import com.mesalab.qgw.model.basic.HttpConfig; +import com.mesalab.qgw.service.impl.HttpClientServiceV2; +import com.mesalab.services.common.entity.KnowledgeConfig; +import com.mesalab.services.common.entity.KnowledgeConstant; +import com.mesalab.services.service.KBService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @Author wxs + * @Date 2022/7/7 + */ +@Component +public class KnowledgeScheduler { + + private static final Log log = LogFactory.get(); + + + @Autowired + private HttpConfig httpConfig; + @Autowired + KnowledgeConfig knowledgeConfig; + @Autowired + private HttpClientServiceV2 httpClientServiceV2; + @Autowired + private KBService kbService; + private static final Map<String, String> updateMap = new HashMap<>(); + + @PostConstruct + private void loadKnowledge() { + try { + BaseResult baseResult = kbService.getList(null, null); + if (!baseResult.isSuccess()) { + log.error("get KB list error: {}", baseResult.getMessage()); + } + List<Map<String, Object>> knowledgeList = (List<Map<String, Object>>) baseResult.getData(); + for (Map<String, Object> knowledge : knowledgeList) { + String kbId = String.valueOf(knowledge.get("kb_id")); + String version = String.valueOf(knowledge.get("version")); + String sha256 = String.valueOf(knowledge.get("sha256")); + String path = String.valueOf(knowledge.get("path")); + if (knowledgeConfig.getIpUserDefined().equals(kbId.concat("_").concat(version)) + && !sha256.equals(updateMap.get(kbId))) { + boolean downloadSucceed = downloadSucceed(path, KnowledgeConstant.DAT + File.separator + KnowledgeConstant.IP_USER_DEFINED + KnowledgeConstant.MMDB); + if (downloadSucceed) { + rebuildIpLookup(); + updateMap.put(kbId, sha256); + } + continue; + } + if (knowledgeConfig.getIpBuiltIn().equals(kbId.concat("_").concat(version)) + && !sha256.equals(updateMap.get(kbId))) { + boolean downloadSucceed = downloadSucceed(path, KnowledgeConstant.DAT + File.separator + KnowledgeConstant.IP_BUILTIN + KnowledgeConstant.MMDB); + if (downloadSucceed) { + rebuildIpLookup(); + updateMap.put(kbId, sha256); + } + continue; + } + if (knowledgeConfig.getIpAsn().equals(kbId.concat("_").concat(version)) + && !sha256.equals(updateMap.get(kbId))) { + boolean downloadSucceed = downloadSucceed(path, KnowledgeConstant.DAT + File.separator + KnowledgeConstant.ASN_BUILTIN + KnowledgeConstant.MMDB); + if (downloadSucceed) { + rebuildIpLookup(); + updateMap.put(kbId, sha256); + } + } + } + } catch (RuntimeException e) { + log.error("loadIpLocation error: {}", e.getMessage()); + } + } + + private static void rebuildIpLookup() { + IPUtil.ipLookupBuild(); + } + + private boolean downloadSucceed(String url, String filePath) { + FileOutputStream outputStream = null; + InputStream inputStream = null; + try { + inputStream = httpClientServiceV2.getInputStream(url, httpConfig.getServerResponseTimeOut(), Maps.newHashMap()); + if (inputStream == null) { + log.error("download file error, skip this update, knowledge is: {}", url); + return false; + } + outputStream = new FileOutputStream(filePath); + IoUtil.copy(inputStream, outputStream); + log.info("knowledge download path is: {}", filePath); + return true; + } catch (IOException e) { + log.error("download file error, message: {}", e.getMessage()); + return false; + } finally { + IoUtil.close(inputStream); + IoUtil.close(outputStream); + } + } +} diff --git a/src/main/java/com/mesalab/qgw/model/basic/AuditServiceLog.java b/src/main/java/com/mesalab/qgw/model/basic/AuditServiceLog.java index aa5748b6..0188743d 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/AuditServiceLog.java +++ b/src/main/java/com/mesalab/qgw/model/basic/AuditServiceLog.java @@ -8,7 +8,7 @@ public class AuditServiceLog { private String annotation; - private String queryKey; + private String requestId; private String cacheKey; @@ -16,11 +16,11 @@ public class AuditServiceLog { private String clientIp; - private long exeTime; + private long elapsed; - private String dbType; + private String dbEngine; - private String param; + private String requestParam; private String url; diff --git a/src/main/java/com/mesalab/qgw/model/basic/ClickHouseHttpSource.java b/src/main/java/com/mesalab/qgw/model/basic/ClickHouseHttpSource.java index d27f290e..7ef769f8 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/ClickHouseHttpSource.java +++ b/src/main/java/com/mesalab/qgw/model/basic/ClickHouseHttpSource.java @@ -25,6 +25,11 @@ public class ClickHouseHttpSource implements Serializable { @NacosValue(value = "${clickhouse.enableApproximateOptimizer}", autoRefreshed = true) private boolean enableApproximateOptimizer; /** + * 是否指定时间为UTC + */ + @NacosValue(value = "${clickhouse.convertDefaultTimezoneIntoUTC}", autoRefreshed = true) + private boolean convertDefaultTimezoneIntoUTC; + /** * realTime的用户名 */ @NacosValue(value = "${clickhouse.realTimeAccount.username}") diff --git a/src/main/java/com/mesalab/qgw/model/basic/ClickHouseHttpQuery.java b/src/main/java/com/mesalab/qgw/model/basic/ClickHouseQueryParam.java index 55e84898..f6610cf7 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/ClickHouseHttpQuery.java +++ b/src/main/java/com/mesalab/qgw/model/basic/ClickHouseQueryParam.java @@ -9,8 +9,9 @@ import lombok.Data; * @Description : ClickHouseHttpQuery */ @Data -public class ClickHouseHttpQuery { - //查询参数 +@Deprecated +public class ClickHouseQueryParam { + @Deprecated private String queryParameter; //响应超时时间 private int socketTimeOut; diff --git a/src/main/java/com/mesalab/qgw/model/basic/CommonRequestParam.java b/src/main/java/com/mesalab/qgw/model/basic/CommonRequestParam.java new file mode 100644 index 00000000..11d8055b --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/basic/CommonRequestParam.java @@ -0,0 +1,57 @@ +package com.mesalab.qgw.model.basic; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.EnumMultiset; +import com.mesalab.common.enums.ExecutionMode; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.enums.OutputMode; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.qgw.exception.QGWBusinessException; +import lombok.Data; + +import java.io.Serializable; +import java.util.Arrays; + +/** + * Common request parameters for all query types, including: SQL and DSL. + * @Date 2023/12/6 15:30 + * @Author wWei + */ +@Data +public class CommonRequestParam implements Serializable { + private String id; + @JsonProperty("output_mode") + private OutputMode outputMode = OutputMode.JSON; + @JsonProperty("execution_mode") + private ExecutionMode executionMode = ExecutionMode.NORMAL; + @JsonProperty("is_dry_run") + private boolean isDryRun = false; + @JsonProperty("is_saved_query") + private boolean isSavedQuery = false; + @JsonProperty("timeout") + private Integer timeout; + + public void setOutputMode(String outputMode) { + OutputMode[] values = OutputMode.values(); + for (OutputMode value : values) { + if(value.getValue().equalsIgnoreCase(outputMode)){ + outputMode = value.name(); + break; + } + } + this.outputMode = OutputMode.valueOf(outputMode.toUpperCase()); + } + + public void setExecutionMode(String executionMode) { + this.executionMode = ExecutionMode.valueOf(executionMode.toUpperCase()); + } + + public void setIsDryRun(Integer isDryRun) { + this.isDryRun = isDryRun != 0; + } + + public void setIsSavedQuery(Integer isSavedQuery) { + this.isSavedQuery = isSavedQuery != 0; + } +}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/qgw/model/basic/DSLProfile.java b/src/main/java/com/mesalab/qgw/model/basic/DSLQueryContext.java index bad149fc..ce3a7894 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/DSLProfile.java +++ b/src/main/java/com/mesalab/qgw/model/basic/DSLQueryContext.java @@ -18,7 +18,7 @@ import java.util.Objects; @Builder @NoArgsConstructor @AllArgsConstructor -public class DSLProfile implements Serializable { +public class DSLQueryContext implements Serializable { private String name; private String granularity; @@ -38,6 +38,8 @@ public class DSLProfile implements Serializable { sql = StrUtil.isNotEmpty(table) ? sql.replace("$table", table) : sql; + sql = !CollectionUtils.isEmpty(this.intervals) ? sql.replace("$interval_seconds", Long.toString(getIntervalSeconds(this.intervals))) : sql; + if (StrUtil.isEmpty(this.filter) && CollectionUtils.isEmpty(this.intervals)) { sql = sql.replace("$intervals_and_filter", " 1= 1 "); } else if (StrUtil.isNotEmpty(this.filter) && !CollectionUtils.isEmpty(this.intervals)) { @@ -54,6 +56,16 @@ public class DSLProfile implements Serializable { return sql; } + private long getIntervalSeconds(List<String> intervals) { + if (CollectionUtils.isEmpty(intervals) || !intervals.get(0).contains("/")) { + throw new IllegalArgumentException(); + } + String[] split = intervals.get(0).split("/"); + long start = DateUtil.parse(split[0]).getTime() / 1000; + long end = DateUtil.parse(split[1]).getTime() / 1000; + return end - start; + } + private String parseIntervals(List<String> intervals, String timeField, String timeFieldType) { if (CollectionUtils.isEmpty(intervals) || !intervals.get(0).contains("/")) { return null; @@ -64,7 +76,7 @@ public class DSLProfile implements Serializable { StringBuffer whereOfTime = new StringBuffer(); if ("unix_timestamp".equalsIgnoreCase(timeFieldType)) { return whereOfTime.append(timeField).append(">= ").append(DateUtil.parse(start).getTime() / 1000).append(" AND ").append(timeField).append("< ").append(DateUtil.parse(end).getTime() / 1000).toString(); - } else{ + } else { return whereOfTime.append(timeField).append(">= '").append(start).append("' AND ").append(timeField).append("< '").append(end).append("'").toString(); } } diff --git a/src/main/java/com/mesalab/qgw/model/basic/DSLQueryRequestParam.java b/src/main/java/com/mesalab/qgw/model/basic/DSLQueryRequestParam.java new file mode 100644 index 00000000..8dbcad35 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/basic/DSLQueryRequestParam.java @@ -0,0 +1,47 @@ +package com.mesalab.qgw.model.basic; + +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Maps; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +/** + * TODO + * + * @Classname DslQueryRequest + * @Date 2023/12/6 15:32 + * @Author wWei + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class DSLQueryRequestParam extends CommonRequestParam implements Serializable { + @JsonProperty("name") + private String name; + @JsonProperty("data_source") + private String dataSource; + @JsonProperty("granularity") + private String granularity; + @JsonProperty("filter") + private String filter; + @JsonProperty("order_by") + private String orderBy; + @JsonProperty("intervals") + private List<String> intervals; + @JsonProperty("limit") + private String limit; + private Map<String, Object> customRequestParam = Maps.newHashMap(); + + @JsonAnySetter + public void setCustomAttributes(String key, Object value) { + customRequestParam.put(key, value); + } +} diff --git a/src/main/java/com/mesalab/qgw/model/basic/DruidIoHttpSource.java b/src/main/java/com/mesalab/qgw/model/basic/DruidIoHttpSource.java index 7dfc18ac..2230d8c0 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/DruidIoHttpSource.java +++ b/src/main/java/com/mesalab/qgw/model/basic/DruidIoHttpSource.java @@ -14,4 +14,5 @@ public class DruidIoHttpSource implements Serializable { private String url; private String dbname; private String skipEmptyBuckets = "true"; + private Integer maxSubqueryRows; } diff --git a/src/main/java/com/mesalab/qgw/model/basic/EngineConfigSource.java b/src/main/java/com/mesalab/qgw/model/basic/EngineConfigSource.java index 025d8ca4..134e36b2 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/EngineConfigSource.java +++ b/src/main/java/com/mesalab/qgw/model/basic/EngineConfigSource.java @@ -1,7 +1,6 @@ package com.mesalab.qgw.model.basic; -import com.alibaba.nacos.api.config.ConfigType; -import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; +import com.alibaba.nacos.api.config.annotation.NacosValue; import lombok.Data; import org.springframework.stereotype.Component; @@ -9,10 +8,18 @@ import java.io.Serializable; @Data @Component -@NacosConfigurationProperties(prefix = "engine", dataId = "${nacos.config.data-id}", groupId = "${nacos.config.group}", type = ConfigType.YAML, autoRefreshed = true) public class EngineConfigSource implements Serializable { + @NacosValue(value = "${engine.maxCacheNum}", autoRefreshed = true) private int maxCacheNum; + @NacosValue(value = "${engine.defaultResultNum}", autoRefreshed = true) private int defaultResultNum; - + @NacosValue(value = "${engine.fieldDiscovery.topK}", autoRefreshed = true) + private int fieldDiscoveryTopK; + @NacosValue(value = "${engine.packetCombine.delaySeconds}", autoRefreshed = true) + private int packetCombineDelaySeconds; + @NacosValue(value = "${engine.trafficSpectrum.clientIPAppResultNum}", autoRefreshed = true) + private int trafficSpectrumClientIPAppResultNum; + @NacosValue(value = "${engine.highLatency.threshold}", autoRefreshed = true) + private int highLatencyThreshold; } diff --git a/src/main/java/com/mesalab/qgw/model/basic/EntityConfigSource.java b/src/main/java/com/mesalab/qgw/model/basic/EntityConfigSource.java deleted file mode 100644 index 1c5c2d7e..00000000 --- a/src/main/java/com/mesalab/qgw/model/basic/EntityConfigSource.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.mesalab.qgw.model.basic; - -import com.alibaba.nacos.api.config.annotation.NacosValue; -import lombok.Data; -import org.springframework.stereotype.Component; - -/** - * 实体推荐默认num - * - * @Classname EntityConfigSource - * @Date 2021/11/29 10:02 下午 - * @Author wWei - */ -@Data -@Component -public class EntityConfigSource { - - /** - * 默认活跃客户端IP数量 - */ - @NacosValue(value = "${service.entity.activeClientIP}", autoRefreshed = true) - private int activeClientIP; - - /** - * 默认活跃服务端IP数量(TCPBySession) - */ - @NacosValue(value = "${service.entity.topServerIP.TCPBySession}", autoRefreshed = true) - private int topServerIPByTCPSession; - - /** - * 默认活跃服务端IP数量(TCPByUniqClientIP) - */ - @NacosValue(value = "${service.entity.topServerIP.TCPByUniqClientIP}", autoRefreshed = true) - private int topServerIPByTCPUniqClientIP; - - /** - * 默认活跃服务端IP数量(UDPBySession) - */ - @NacosValue(value = "${service.entity.topServerIP.UDPBySession}", autoRefreshed = true) - private int topServerIPByUDPBySession; - - /** - * 默认活跃服务端IP数量(UDPByUniqClientIP) - */ - @NacosValue(value = "${service.entity.topServerIP.UDPByUniqClientIP}", autoRefreshed = true) - private int topServerIPByUDPUniqClientIP; - - /** - * 默认访问量最高SNI数量 - */ - @NacosValue(value = "${service.entity.topSNI.defaultSize}", autoRefreshed = true) - private int topSNIDefaultSize; - - /** - * 待推荐SNI数据集 - */ - @NacosValue(value = "${service.entity.topSNI.dataset}", autoRefreshed = true) - private int topSNIDataset; - - /** - * 待推荐GTP-C - */ - @NacosValue(value = "${service.entity.gtpc.defaultSize}", autoRefreshed = true) - private int topGTPCDefaultSize; - -} diff --git a/src/main/java/com/mesalab/qgw/model/basic/HttpResponseResult.java b/src/main/java/com/mesalab/qgw/model/basic/HttpResponseResult.java new file mode 100644 index 00000000..0f2ad442 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/basic/HttpResponseResult.java @@ -0,0 +1,22 @@ +package com.mesalab.qgw.model.basic; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.Map; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode +public class HttpResponseResult implements Serializable { + private Integer statusCode; + private String errorMessage; + private Map<String, String> responseHeaders; + private String responseBody; + + +} diff --git a/src/main/java/com/mesalab/qgw/model/basic/QueryCache.java b/src/main/java/com/mesalab/qgw/model/basic/QueryCache.java new file mode 100644 index 00000000..aed03fd1 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/basic/QueryCache.java @@ -0,0 +1,40 @@ +package com.mesalab.qgw.model.basic; + +import com.google.common.collect.Maps; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.services.configuration.JobConfig; +import lombok.Data; + +import java.io.Serializable; +import java.util.Map; + +/** + * TODO + * + * @Classname QueryJobInfo + * @Date 2023/12/6 18:20 + * @Author wWei + */ +@Data +public class QueryCache implements Serializable { + private String type; + private Long latestQueryTimeMs = System.currentTimeMillis(); + private BaseResult<Object> baseResult; + public QueryCache(String jobId) { + this.baseResult = new BaseResult<>(); + Map<String, Object> job = Maps.newLinkedHashMap(); + job.put(JobConfig.JOB_ID, jobId); + job.put(JobConfig.IS_DONE, false); + job.put(JobConfig.DONE_PROGRESS, 0.0); + job.put(JobConfig.IS_CANCELED, false); + job.put(JobConfig.IS_FAILED, false); + job.put(JobConfig.REASON, null); + job.put(JobConfig.START_TIME, null); + job.put(JobConfig.END_TIME, null); + Map<String, Object> links = Maps.newLinkedHashMap(); + links.put(JobConfig.LINKS_STATUS, "/v1/query/job/" + jobId); + links.put(JobConfig.LINKS_RESULT, "/v1/query/job/" + jobId + "/result"); + job.put(JobConfig.LINKS, links); + this.baseResult.setJob(job); + } +} diff --git a/src/main/java/com/mesalab/qgw/model/basic/QueryProfile.java b/src/main/java/com/mesalab/qgw/model/basic/QueryProfile.java deleted file mode 100644 index 0c256fbd..00000000 --- a/src/main/java/com/mesalab/qgw/model/basic/QueryProfile.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.mesalab.qgw.model.basic; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.geedgenetworks.utils.StringUtil; -import com.mesalab.common.enums.DBTypeEnum; -import lombok.*; -import java.io.Serializable; -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class QueryProfile implements Serializable { - - private String query; - private String option; - private Boolean sampled; - private String format; - private String queryId; - private String resultId; - /** - * @see DBTypeEnum - */ - private String dbType; - /** - * 方言数据库类型，用于SQL格式解析校验 - */ - private String dialectDBType; - private SQLQuerySource engineQuerySource; - private SQLQuerySource dbQuerySource; - - public String getResultId() { - return resultId; - } - - public boolean queryParamIsNull() { - return StringUtil.isBlank(query); - } - - public void setResult_id(String resultId) { - this.resultId = resultId; - } -} diff --git a/src/main/java/com/mesalab/qgw/model/basic/SQLQueryContext.java b/src/main/java/com/mesalab/qgw/model/basic/SQLQueryContext.java new file mode 100644 index 00000000..634ff1a6 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/basic/SQLQueryContext.java @@ -0,0 +1,35 @@ +package com.mesalab.qgw.model.basic; + +import com.geedgenetworks.utils.StringUtil; +import lombok.*; + +import java.io.Serializable; +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SQLQueryContext implements Serializable { + + private String originalSQL; + private String option; + private Boolean sampled; + private String format; + private String queryId; + private String resultId; + private Integer timeout; + private String dbEngine; + private String sqlDialect; // SQL Dialect,such as: Mysql,Hive, etc. It used syntax and semantic rules to transform SQL query. + private SelectStatement federationSelectStatement; + private SelectStatement dbSelectStatement; + public String getResultId() { + return resultId; + } + + public boolean originalSQLIsNull() { + return StringUtil.isBlank(originalSQL); + } + + public void setResultId(String resultId) { + this.resultId = resultId; + } +} diff --git a/src/main/java/com/mesalab/qgw/model/basic/SQLQuerySource.java b/src/main/java/com/mesalab/qgw/model/basic/SQLQuerySource.java deleted file mode 100644 index d125b47a..00000000 --- a/src/main/java/com/mesalab/qgw/model/basic/SQLQuerySource.java +++ /dev/null @@ -1,82 +0,0 @@ -package com.mesalab.qgw.model.basic; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.mesalab.common.utils.sqlparser.SQLHelper; -import com.mesalab.qgw.model.basic.udf.UDF; -import com.geedgenetworks.utils.StringUtil; -import lombok.Data; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.statement.select.*; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * SQL Parser properties - */ -@Data -public class SQLQuerySource { - - private String execSQL; - private String sqlBody; - private List<String> tableNames = Lists.newArrayList(); - private String partitionKey; - private List<Join> joins; - private FromItem fromItem; - private List<SelectItem> selectItems; - private Map<String, String> aliasFields = Maps.newHashMap(); - private Expression whereExpression; - private GroupByElement groupByElement; - private List<OrderByElement> orderByElements; - private Map<String, String> groupDimension = Maps.newHashMap(); - private String limit; - private boolean isEnableLimit; - - private SubSelect subSelect; - private List<SQLQuerySource> subSqlQuerySources = Lists.newArrayList(); - private Set<UDF> udfSet = Sets.newHashSet(); - - public boolean isEnableLimit() { - return !SQLHelper.INVALID_LIMIT_DESC.equalsIgnoreCase(limit); - } - - public String getSelect() { - return Joiner.on(",").join(this.selectItems); - } - - public String getOrderBy() { - return Joiner.on(",").join(this.orderByElements); - } - public Map<String, String> getGroupDimension() { - if (groupByElement == null) { - return null; - } - groupBy: - for (Expression groupByExpression : groupByElement.getGroupByExpressions()) { - - String groupBy = groupByExpression.toString(); - if ((groupBy.startsWith("\"") && groupBy.endsWith("\"")) - || ((groupBy.startsWith("`") && groupBy.endsWith("`")))) { - groupBy = groupBy.substring(1, groupBy.length() - 1); - } - if (StringUtil.isNotBlank(aliasFields.get(groupBy))) { - groupDimension.put(groupBy, aliasFields.get(groupBy)); - } else { - //非别名，确定实际的别名 - for (String key : aliasFields.keySet()) { - if (aliasFields.get(key).equalsIgnoreCase(groupBy)) { - groupDimension.put(key, groupBy); - continue groupBy; - } - } - groupDimension.put(groupBy, groupBy); - } - - } - return groupDimension; - } -} diff --git a/src/main/java/com/mesalab/qgw/model/basic/SelectStatement.java b/src/main/java/com/mesalab/qgw/model/basic/SelectStatement.java new file mode 100644 index 00000000..c3d1c82f --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/basic/SelectStatement.java @@ -0,0 +1,103 @@ +package com.mesalab.qgw.model.basic; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.mesalab.common.utils.sqlparser.SQLFunctionUtil; +import com.mesalab.common.utils.sqlparser.SQLHelper; +import com.mesalab.qgw.model.basic.udf.UDF; +import com.geedgenetworks.utils.StringUtil; +import lombok.Data; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.statement.select.*; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A parsed SQL statement + */ +@Data +public class SelectStatement { + private String execSQL; + private String sqlBody; + private List<String> tableNames = Lists.newArrayList(); + private String partitionKey; + private List<Join> joins; + private FromItem fromItem; + private List<SelectItem> selectItems; + private Map<String, String> aliasFields = Maps.newHashMap(); + private Expression whereExpression; + private GroupByElement groupByElement; + private List<OrderByElement> orderByElements; + private Map<String, String> groupDimension = Maps.newHashMap(); + private String limit; + private boolean isEnableLimit; + + private SubSelect subSelect; + private List<SelectStatement> subSqlQuerySources = Lists.newArrayList(); + private Set<UDF> udfSet = Sets.newHashSet(); + + public boolean isEnableLimit() { + return !SQLHelper.INVALID_LIMIT_DESC.equalsIgnoreCase(limit); + } + + public String getSelect() { + return Joiner.on(",").join(this.selectItems); + } + + public String getOrderBy() { + return Joiner.on(",").join(this.orderByElements); + } + + public Map<String, String> getGroupDimension() { + if (groupByElement == null) { + return Maps.newHashMap(); + } + ExpressionList groupByExpressionList = groupByElement.getGroupByExpressionList(); + + groupBy: + for (Expression expression : groupByExpressionList.getExpressions()) { + String groupBy = expression.toString(); + groupBy = SQLHelper.removeQuotesAndBackticks(groupBy); + if (expression instanceof Function && SQLFunctionUtil.ROLLUP.equalsIgnoreCase(((Function) expression).getName())) { + List<Expression> funParamList = ((Function) expression).getParameters().getExpressions(); + paramGroupBy: + for (Expression param : funParamList) { + String paramGroupBy = param.toString(); + paramGroupBy = SQLHelper.removeQuotesAndBackticks(paramGroupBy); + if (StringUtil.isNotBlank(aliasFields.get(paramGroupBy))) { + groupDimension.put(paramGroupBy, aliasFields.get(paramGroupBy)); + } else { + //非别名，确定实际的别名 + for (String key : aliasFields.keySet()) { + if (aliasFields.get(key).equalsIgnoreCase(paramGroupBy)) { + groupDimension.put(key, paramGroupBy); + continue paramGroupBy; + } + } + groupDimension.put(paramGroupBy, paramGroupBy); + } + } + } else { + if (StringUtil.isNotBlank(aliasFields.get(groupBy))) { + groupDimension.put(groupBy, aliasFields.get(groupBy)); + } else { + //非别名，确定实际的别名 + for (String key : aliasFields.keySet()) { + if (aliasFields.get(key).equalsIgnoreCase(groupBy)) { + groupDimension.put(key, groupBy); + continue groupBy; + } + } + groupDimension.put(groupBy, groupBy); + } + } + } + return groupDimension; + } +} diff --git a/src/main/java/com/mesalab/qgw/model/basic/SqlQueryRequestParam.java b/src/main/java/com/mesalab/qgw/model/basic/SqlQueryRequestParam.java new file mode 100644 index 00000000..a58063be --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/basic/SqlQueryRequestParam.java @@ -0,0 +1,31 @@ +package com.mesalab.qgw.model.basic; + +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.google.common.collect.Maps; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.Map; + +/** + * + * HTTP Rest API request parameters for SQL query. + * @Date 2023/12/6 15:32 + * @Author wWei + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class SqlQueryRequestParam extends CommonRequestParam implements Serializable { + private String statement; + private Map<String, Object> customRequestParam = Maps.newHashMap(); + + @JsonAnySetter + public void setCustomAttributes(String key, Object value) { + customRequestParam.put(key, value); + } +} diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN.java b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN.java index dc2c05e9..063d6a00 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN.java +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN.java @@ -3,7 +3,7 @@ package com.mesalab.qgw.model.basic.udf; import com.google.common.collect.Lists; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.utils.IPUtil; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.qgw.model.basic.SelectStatement; import com.geedgenetworks.utils.StringUtil; import lombok.AllArgsConstructor; import lombok.Data; @@ -28,15 +28,15 @@ public class IP_TO_ASN implements UDF { } @Override - public Object execute(SQLQuerySource sqlQuerySource, BaseResult baseResult) { + public Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult) { List<Object> results = (List<Object>)baseResult.getData(); List<Map<String, Object>> targetResult = Lists.newArrayList(); String addressLabel = StringUtil.EMPTY; - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name) ) { + for (String alias : selectStatement.getAliasFields().keySet()) { + if (selectStatement.getAliasFields().get(alias).contains(address) && + selectStatement.getAliasFields().get(alias).contains(name) ) { addressLabel = alias; break; } diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN_DETAIL.java b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN_DETAIL.java index 7cfd4f48..f98fa450 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN_DETAIL.java +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN_DETAIL.java @@ -3,7 +3,7 @@ package com.mesalab.qgw.model.basic.udf; import com.google.common.collect.Lists; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.utils.IPUtil; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.qgw.model.basic.SelectStatement; import com.geedgenetworks.utils.StringUtil; import lombok.AllArgsConstructor; import lombok.Data; @@ -28,15 +28,15 @@ public class IP_TO_ASN_DETAIL implements UDF { } @Override - public Object execute(SQLQuerySource sqlQuerySource, BaseResult baseResult) { + public Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult) { List<Object> results = (List<Object>)baseResult.getData(); List<Map<String, Object>> targetResult = Lists.newArrayList(); String addressLabel = StringUtil.EMPTY; - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name) ) { + for (String alias : selectStatement.getAliasFields().keySet()) { + if (selectStatement.getAliasFields().get(alias).contains(address) && + selectStatement.getAliasFields().get(alias).contains(name) ) { addressLabel = alias; break; } diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN_ORG.java b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN_ORG.java index 442bd417..9d55afbc 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN_ORG.java +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ASN_ORG.java @@ -3,7 +3,7 @@ package com.mesalab.qgw.model.basic.udf; import com.google.common.collect.Lists; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.utils.IPUtil; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.qgw.model.basic.SelectStatement; import com.geedgenetworks.utils.StringUtil; import lombok.AllArgsConstructor; import lombok.Data; @@ -28,15 +28,15 @@ public class IP_TO_ASN_ORG implements UDF { } @Override - public Object execute(SQLQuerySource sqlQuerySource, BaseResult baseResult) { + public Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult) { List<Object> results = (List<Object>)baseResult.getData(); List<Map<String, Object>> targetResult = Lists.newArrayList(); String addressLabel = StringUtil.EMPTY; - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name) ) { + for (String alias : selectStatement.getAliasFields().keySet()) { + if (selectStatement.getAliasFields().get(alias).contains(address) && + selectStatement.getAliasFields().get(alias).contains(name) ) { addressLabel = alias; break; } diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_CITY.java b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_CITY.java index f0678427..8bbc0e0a 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_CITY.java +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_CITY.java @@ -3,7 +3,7 @@ package com.mesalab.qgw.model.basic.udf; import com.google.common.collect.Lists; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.utils.IPUtil; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.qgw.model.basic.SelectStatement; import com.geedgenetworks.utils.StringUtil; import lombok.AllArgsConstructor; import lombok.Data; @@ -28,14 +28,14 @@ public class IP_TO_CITY implements UDF { } @Override - public Object execute(SQLQuerySource sqlQuerySource, BaseResult baseResult) { + public Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult) { List<Object> results = (List<Object>)baseResult.getData(); List<Map<String, Object>> targetResult = Lists.newArrayList(); String addressLabel = StringUtil.EMPTY; - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name) ) { + for (String alias : selectStatement.getAliasFields().keySet()) { + if (selectStatement.getAliasFields().get(alias).contains(address) && + selectStatement.getAliasFields().get(alias).contains(name) ) { addressLabel = alias; break; } diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_COUNTRY.java b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_COUNTRY.java index 042ac858..7b919a97 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_COUNTRY.java +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_COUNTRY.java @@ -3,7 +3,7 @@ package com.mesalab.qgw.model.basic.udf; import com.google.common.collect.Lists; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.utils.IPUtil; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.qgw.model.basic.SelectStatement; import com.geedgenetworks.utils.StringUtil; import lombok.AllArgsConstructor; import lombok.Data; @@ -28,13 +28,13 @@ public class IP_TO_COUNTRY implements UDF { } @Override - public Object execute(SQLQuerySource sqlQuerySource, BaseResult baseResult) { + public Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult) { List<Object> results = (List<Object>)baseResult.getData(); List<Map<String, Object>> targetResult = Lists.newArrayList(); String addressLabel = StringUtil.EMPTY; - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name) ) { + for (String alias : selectStatement.getAliasFields().keySet()) { + if (selectStatement.getAliasFields().get(alias).contains(address) && + selectStatement.getAliasFields().get(alias).contains(name) ) { addressLabel = alias; break; } diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_GEO.java b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_GEO.java index caabeac2..7f5107fb 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_GEO.java +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_GEO.java @@ -3,7 +3,7 @@ package com.mesalab.qgw.model.basic.udf; import com.google.common.collect.Lists; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.utils.IPUtil; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.qgw.model.basic.SelectStatement; import com.geedgenetworks.utils.StringUtil; import lombok.AllArgsConstructor; import lombok.Data; @@ -27,15 +27,15 @@ public class IP_TO_GEO implements UDF { } @Override - public Object execute(SQLQuerySource sqlQuerySource, BaseResult baseResult) { + public Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult) { List<Object> results = (List<Object>)baseResult.getData(); List<Map<String, Object>> targetResult = Lists.newArrayList(); String addressLabel = StringUtil.EMPTY; - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name) ) { + for (String alias : selectStatement.getAliasFields().keySet()) { + if (selectStatement.getAliasFields().get(alias).contains(address) && + selectStatement.getAliasFields().get(alias).contains(name) ) { addressLabel = alias; break; } diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ISP.java b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ISP.java index 59c28e61..e73d3f98 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ISP.java +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/IP_TO_ISP.java @@ -3,7 +3,7 @@ package com.mesalab.qgw.model.basic.udf; import com.google.common.collect.Lists; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.utils.IPUtil; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.qgw.model.basic.SelectStatement; import com.geedgenetworks.utils.StringUtil; import lombok.AllArgsConstructor; import lombok.Data; @@ -28,15 +28,15 @@ public class IP_TO_ISP implements UDF { } @Override - public Object execute(SQLQuerySource sqlQuerySource, BaseResult baseResult) { + public Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult) { List<Object> results = (List<Object>)baseResult.getData(); List<Map<String, Object>> targetResult = Lists.newArrayList(); String addressLabel = StringUtil.EMPTY; - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name) ) { + for (String alias : selectStatement.getAliasFields().keySet()) { + if (selectStatement.getAliasFields().get(alias).contains(address) && + selectStatement.getAliasFields().get(alias).contains(name) ) { addressLabel = alias; break; } diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/ROLLUP.java b/src/main/java/com/mesalab/qgw/model/basic/udf/ROLLUP.java new file mode 100644 index 00000000..23ec8e84 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/ROLLUP.java @@ -0,0 +1,114 @@ +package com.mesalab.qgw.model.basic.udf; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.DataTypeMapping; +import com.mesalab.common.utils.sqlparser.SQLHelper; +import com.mesalab.qgw.constant.MetaConst; +import com.mesalab.qgw.model.basic.SelectStatement; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ROLLUP implements UDF { + private String name; + private String regularDimension; + private String drillDownDimension; + + + public ROLLUP(UDFElements udfElements) { + this.name = udfElements.getName(); + this.regularDimension = SQLHelper.removeQuotesAndBackticks(udfElements.getParams().get(0).toString()); + this.drillDownDimension = SQLHelper.removeQuotesAndBackticks(udfElements.getParams().get(1).toString()); + } + + @Override + public Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult) { + List<Map<String, String>> meta = (List<Map<String, String>>) baseResult.getMeta(); + for (Map<String, String> stringStringMap : meta) { + if (stringStringMap.get(MetaConst.META_CATEGORY).equals(MetaConst.META_CATEGORY_DIMENSION)) { + stringStringMap.put(MetaConst.META_TYPE, DataTypeMapping.STRING); + } + } + List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); + + // TODO: 样例数据时，需要补充subtotal rows + if (isDryRun) { + List<Map<String, Object>> newData = new ArrayList<>(); + newData.add(Maps.newLinkedHashMap()); + for (Map<String, Object> element : data) { + Map<String, Object> newElement = new LinkedHashMap<>(element); + newElement.forEach((k, v) -> { + if (drillDownDimension.equals(k) || drillDownDimension.equals(selectStatement.getAliasFields().get(k))) { + newElement.put(k, null); + } + }); + newData.add(newElement); + newData.add(element); + } + data = newData; + } + // TODO: 遍历data 将rollup数据集结果标记subtotal、并且删除total总计行 + Iterator<Map<String, Object>> iterator = data.iterator(); + HashSet<Object> drillDownDimensionSets = Sets.newHashSet(); + AtomicBoolean hasRemoveRegularDimension = new AtomicBoolean(false); + + while (iterator.hasNext()) { + Map<String, Object> next = iterator.next(); + if (!hasRemoveRegularDimension.get()) { + // TODO 删除total rows + if (meta.stream().filter(o -> MetaConst.META_CATEGORY_DIMENSION.equalsIgnoreCase(o.get(MetaConst.META_CATEGORY))).allMatch(o -> getRollupRowsDefaultValues().contains(next.get(o.get(MetaConst.META_NAME))))) { + iterator.remove(); + hasRemoveRegularDimension.set(true); + continue; + } + } + next.forEach((k, v) -> { + if (drillDownDimension.equals(k) || drillDownDimension.equals(selectStatement.getAliasFields().get(k))) { + Object regularDimensionValue = next.get(regularDimension); + if (!next.containsKey(regularDimension)) { + Set<String> aliasSet = selectStatement.getAliasFields().keySet(); + for (String alias : aliasSet) { + if (selectStatement.getAliasFields().get(alias).equals(regularDimension)) { + regularDimensionValue = next.get(alias); + break; + } + } + } + if (getRollupRowsDefaultValues().contains(next.get(k)) && (!drillDownDimensionSets.contains(regularDimensionValue))) { + next.put(k, "Subtotal"); + drillDownDimensionSets.add(regularDimensionValue); + } + } + if (meta.stream().anyMatch(o -> o.get(MetaConst.META_NAME).equals(k) + && MetaConst.META_CATEGORY_DIMENSION.equals(o.get(MetaConst.META_CATEGORY))) + && next.get(k) != null + && !(next.get(k) instanceof List)) { + next.put(k, String.valueOf(next.get(k))); + } + }); + } + return data; + } + + private static List<Object> getRollupRowsDefaultValues() { + List<Object> rollupValues = new ArrayList<>(); + rollupValues.add(""); + rollupValues.add(0); + rollupValues.add(null); + rollupValues.add(Lists.newArrayList()); + rollupValues.add("1970-01-01 00:00:00"); + rollupValues.add("1970-01-01 00:00:00.000"); + return rollupValues; + } +} diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/TIME_FLOOR_WITH_FILL.java b/src/main/java/com/mesalab/qgw/model/basic/udf/TIME_FLOOR_WITH_FILL.java index 43eb0015..3a144420 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/udf/TIME_FLOOR_WITH_FILL.java +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/TIME_FLOOR_WITH_FILL.java @@ -15,8 +15,8 @@ import com.mesalab.common.utils.sqlparser.SQLFunctionUtil; import com.mesalab.common.utils.SpringContextUtil; import com.mesalab.common.utils.sqlparser.SQLHelper; import com.mesalab.qgw.model.basic.EngineConfigSource; -import com.mesalab.qgw.model.basic.SQLQuerySource; -import com.mesalab.qgw.service.MetadataService; +import com.mesalab.qgw.model.basic.SelectStatement; +import com.mesalab.qgw.service.DatabaseService; import com.geedgenetworks.utils.DateUtils; import com.geedgenetworks.utils.StringUtil; import lombok.AllArgsConstructor; @@ -39,7 +39,7 @@ import java.util.*; @NoArgsConstructor public class TIME_FLOOR_WITH_FILL implements UDF{ private static final Log log = LogFactory.get(); - private MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("metadataService"); + private DatabaseService databaseService = (DatabaseService) SpringContextUtil.getBean("databaseService"); private EngineConfigSource engineConfigSource = (EngineConfigSource) SpringContextUtil.getBean("engineConfigSource"); private String name; private String timestamp; @@ -90,13 +90,16 @@ public class TIME_FLOOR_WITH_FILL implements UDF{ @Override - public Object execute(SQLQuerySource sqlQuerySource, BaseResult baseResult) { + public Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult) { List<Object> results = (List<Object>)baseResult.getData(); + if(isDryRun) { + return results; + } List<Map<String, Object>> targetResult = Lists.newArrayList(); List<String> groupLabels = Lists.newArrayList(); String timeGranLabel = null; try { - Map<String, String> groupDimension = sqlQuerySource.getGroupDimension(); + Map<String, String> groupDimension = selectStatement.getGroupDimension(); if (results.isEmpty() || StringUtil.isBlank(fill) || StringUtil.isEmpty(groupDimension)) { return results; } @@ -125,7 +128,7 @@ public class TIME_FLOOR_WITH_FILL implements UDF{ return results; } - WhereTimeRange whereTimeRange = getWhereTimeRange(sqlQuerySource); + WhereTimeRange whereTimeRange = getWhereTimeRange(selectStatement); Date benchmarkDate = getBenchmarkDate(resultsFirst, timeGranLabel); targetResult = fillResultBasedOnTimeSeries(results, groupLabels, timeGranLabel); @@ -168,7 +171,7 @@ public class TIME_FLOOR_WITH_FILL implements UDF{ groupLabels, timeGranLabel); if (StringUtil.isNotEmpty(dateRangeList)) { for (String key : targetTimeSeriesMap.keySet()) { - targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, fill.toString())); + targetTimeSeriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeSeriesMap.get(key), dateRangeList, fill)); } } targetResult = ConvertUtil.convertTimeSeriesToList(targetTimeSeriesMap, groupLabels, timeGranLabel); @@ -176,13 +179,13 @@ public class TIME_FLOOR_WITH_FILL implements UDF{ log.error("Parser function : TIME_FLOOR_WITH_FILL error :{} ", e.toString()); return results; } finally { - if (StringUtil.isNotBlank(sqlQuerySource.getLimit()) - && !sqlQuerySource.getLimit().equalsIgnoreCase(SQLHelper.INVALID_LIMIT_DESC) - && targetResult.size() > Integer.parseInt(sqlQuerySource.getLimit())) { - targetResult = targetResult.subList(0, Integer.parseInt(sqlQuerySource.getLimit())); + if (StringUtil.isNotBlank(selectStatement.getLimit()) + && !selectStatement.getLimit().equalsIgnoreCase(SQLHelper.INVALID_LIMIT_DESC) + && targetResult.size() > Integer.parseInt(selectStatement.getLimit())) { + targetResult = targetResult.subList(0, Integer.parseInt(selectStatement.getLimit())); } } - sortByTime(sqlQuerySource, targetResult, timeGranLabel); + sortByTime(selectStatement, targetResult, timeGranLabel); List<String> objects = Lists.newArrayList(); objects.add(timeGranLabel); objects.addAll(groupLabels); @@ -231,7 +234,7 @@ public class TIME_FLOOR_WITH_FILL implements UDF{ }); } - private void sortByTime(SQLQuerySource sqlQuerySource, List<Map<String, Object>> targetResult, String timeGranLabel) { + private void sortByTime(SelectStatement sqlQuerySource, List<Map<String, Object>> targetResult, String timeGranLabel) { if(StringUtil.isEmpty(sqlQuerySource.getOrderByElements())){ return; } @@ -302,8 +305,8 @@ public class TIME_FLOOR_WITH_FILL implements UDF{ * @param sqlQuerySource * @return */ - private WhereTimeRange getWhereTimeRange(SQLQuerySource sqlQuerySource) { - String dbType = metadataService.getDBTypeByTableName(sqlQuerySource.getTableNames().get(0)); + private WhereTimeRange getWhereTimeRange(SelectStatement sqlQuerySource) { + String dbType = databaseService.getDBEngineByTableName(sqlQuerySource.getTableNames().get(0)); WhereTimeRange whereTimeRange = new WhereTimeRange(); List<FillParam> outToInnerList = getFillParams(sqlQuerySource); for (FillParam fillParam : outToInnerList) { @@ -315,9 +318,9 @@ public class TIME_FLOOR_WITH_FILL implements UDF{ return whereTimeRange; } - private List<FillParam> getFillParams(SQLQuerySource sqlQuerySource) { + private List<FillParam> getFillParams(SelectStatement sqlQuerySource) { List<FillParam> result = Lists.newArrayList(); - SQLQuerySource currentSource = sqlQuerySource; + SelectStatement currentSource = sqlQuerySource; boolean isOriginalDataSource = false; while (!isOriginalDataSource) { FillParam currentParam = new FillParam(); diff --git a/src/main/java/com/mesalab/qgw/model/basic/udf/UDF.java b/src/main/java/com/mesalab/qgw/model/basic/udf/UDF.java index 6af3dd10..052f8a70 100644 --- a/src/main/java/com/mesalab/qgw/model/basic/udf/UDF.java +++ b/src/main/java/com/mesalab/qgw/model/basic/udf/UDF.java @@ -1,9 +1,9 @@ package com.mesalab.qgw.model.basic.udf; import com.mesalab.common.entity.BaseResult; -import com.mesalab.qgw.model.basic.SQLQuerySource; +import com.mesalab.qgw.model.basic.SelectStatement; public interface UDF { - Object execute(SQLQuerySource sqlQuerySource, BaseResult baseResult); + Object execute(SelectStatement selectStatement, boolean isDryRun, BaseResult baseResult); } diff --git a/src/main/java/com/mesalab/network/model/protocol/ProtocolTree.java b/src/main/java/com/mesalab/qgw/model/dsl/LiveChartProtocol.java index 27657305..43122b1d 100644 --- a/src/main/java/com/mesalab/network/model/protocol/ProtocolTree.java +++ b/src/main/java/com/mesalab/qgw/model/dsl/LiveChartProtocol.java @@ -1,40 +1,52 @@ -package com.mesalab.network.model.protocol; +package com.mesalab.qgw.model.dsl; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.mesalab.network.common.Constants; +import com.mesalab.qgw.constant.dsl.LiveChartConstants; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; +/** + * @author wangwei + */ @Data @Builder @NoArgsConstructor @AllArgsConstructor -public class ProtocolTree { +@JsonPropertyOrder({"id", "name", "parentId", "childrens", "metrics", "sentBytes", "receivedBytes", "totalSentBytes", "totalReceivedBytes"}) +public class LiveChartProtocol implements Serializable { private String id; private String name; + @JsonProperty("parent_id") private String parentId; - private List<ProtocolTree> childrens = Lists.newArrayList(); + private List<LiveChartProtocol> childrens = Lists.newArrayList(); private Map<String, Object> metrics = Maps.newLinkedHashMap(); + @JsonProperty("sent_bytes") private long sentBytes; + @JsonProperty("received_bytes") private long receivedBytes; + @JsonProperty("total_sent_bytes") private long totalSentBytes; + @JsonProperty("total_received_bytes") private long totalReceivedBytes; - public ProtocolTree(String id, String name, String parentId) { + public LiveChartProtocol(String id, String name, String parentId) { this.id = id; this.name = name; this.parentId = parentId; } - public ProtocolTree(String id, String name, String parentId, long sentBytes, long receivedBytes) { + public LiveChartProtocol(String id, String name, String parentId, long sentBytes, long receivedBytes) { this.id = id; this.name = name; this.parentId = parentId; @@ -48,8 +60,8 @@ public class ProtocolTree { public String getParentId() { - return id.lastIndexOf(Constants.ENCAPSULATION_PATH_SEPARATOR) > 0 ? - id.substring(0, id.lastIndexOf(Constants.ENCAPSULATION_PATH_SEPARATOR)) : null; + return id.lastIndexOf(LiveChartConstants.ENCAPSULATION_PATH_SEPARATOR) > 0 ? + id.substring(0, id.lastIndexOf(LiveChartConstants.ENCAPSULATION_PATH_SEPARATOR)) : null; } public void setParentId(String parentId) { @@ -60,7 +72,7 @@ public class ProtocolTree { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ProtocolTree that = (ProtocolTree) o; + LiveChartProtocol that = (LiveChartProtocol) o; return Objects.equals(id, that.id); } diff --git a/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java index 6b769bdc..1b373112 100644 --- a/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java @@ -57,6 +57,7 @@ public class ArangoHealthIndicator extends AbstractHealthIndicator { StringUtil.EMPTY, new BasicHeader(HttpHeaders.AUTHORIZATION, String.valueOf(jwt))); Map map = JSON.parseObject(result, Map.class); if (StringUtil.isEmpty(map) || StringUtil.isEmpty(map.get("version"))) { + JwtCache.remove(Constant.ARANGO_CACHE_JWT); builder.down() .withDetail("app", APP) .withDetail("message", "连接Arango 失败: " + result); @@ -64,9 +65,11 @@ public class ArangoHealthIndicator extends AbstractHealthIndicator { builder.up() .withDetail("app", APP) .withDetail("url", arangoConfig.getServer()) - .withDetail("message", "ok"); + .withDetail("message", "ok") + .withDetail("version", map.get("version")); } } catch (RuntimeException e) { + JwtCache.remove(Constant.ARANGO_CACHE_JWT); builder.down() .withDetail("app", APP) .withDetail("message", "连接Arango 失败"); diff --git a/src/main/java/com/mesalab/qgw/monitor/ClickHouseHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/ClickHouseHealthIndicator.java index 1b652f8b..758fb86d 100644 --- a/src/main/java/com/mesalab/qgw/monitor/ClickHouseHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/ClickHouseHealthIndicator.java @@ -2,7 +2,7 @@ package com.mesalab.qgw.monitor; import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.URLUtil; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.qgw.model.basic.ClickHouseHttpSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.health.AbstractHealthIndicator; @@ -38,22 +38,24 @@ public class ClickHouseHealthIndicator extends AbstractHealthIndicator { .append(clickHouseHttpSource.getRealTimeAccountUserName()).append("&") .append("password=").append(clickHouseHttpSource.getRealTimeAccountPin()).append("&") .append("database=").append(clickHouseHttpSource.getDbName()).append("&") - .append("query=").append("select timezone() as timeZone, now() as curTimestamp") + .append("query=").append("select version() as version,timezone() as timeZone, now() as timestamp") .append(" FORMAT JSON ;"); String url = queryURL + queryParamBuilder; try { ResponseEntity responseEntity = restTemplate.getForEntity(url, Map.class); - if (responseEntity.getStatusCodeValue() == ResultStatusEnum.SUCCESS.getCode()) { + if (responseEntity.getStatusCodeValue() == HttpStatusCodeEnum.SUCCESS.getCode()) { Map body = (Map) responseEntity.getBody(); Map data = (Map) ((List) body.get("data")).get(0); String dbTimeZone = data.get("timeZone").toString(); - String curDatetime = data.get("curTimestamp").toString(); + String curDatetime = data.get("timestamp").toString(); + String version = StrUtil.subBefore(data.get("version").toString(),".", true); builder.up() .withDetail("app", APP) .withDetail("url", clickHouseHttpSource.getUrl()) .withDetail("message", "ok") .withDetail("timeZone", dbTimeZone) - .withDetail("curTimestamp", curDatetime); + .withDetail("timestamp", curDatetime) + .withDetail("version",version); } else { builder.down() .withDetail("app", APP) diff --git a/src/main/java/com/mesalab/qgw/monitor/DruidHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/DruidHealthIndicator.java index 6ecf528f..061c8ce7 100644 --- a/src/main/java/com/mesalab/qgw/monitor/DruidHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/DruidHealthIndicator.java @@ -4,7 +4,7 @@ import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.URLUtil; import com.alibaba.fastjson2.JSON; import com.google.common.collect.Maps; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.qgw.model.basic.DruidIoHttpSource; import lombok.Data; import org.springframework.beans.factory.annotation.Autowired; @@ -49,7 +49,7 @@ public class DruidHealthIndicator extends AbstractHealthIndicator { HttpEntity request = new HttpEntity(JSON.toJSONString(druidQueryParam), headers); ResponseEntity responseEntity = restTemplate.postForEntity(queryURL, request, Object.class); - if (responseEntity.getStatusCodeValue() == ResultStatusEnum.SUCCESS.getCode()) { + if (responseEntity.getStatusCodeValue() == HttpStatusCodeEnum.SUCCESS.getCode()) { List body = (List) responseEntity.getBody(); String timeZone = String.valueOf(((Map) (body.get(0))).get("timeZone")); String curDatetime = String.valueOf(((Map) (body.get(0))).get("curTimestamp")); @@ -58,7 +58,7 @@ public class DruidHealthIndicator extends AbstractHealthIndicator { .withDetail("url", druidIoHttpSource.getUrl()) .withDetail("message", "ok") .withDetail("timeZone", timeZone) - .withDetail("curTimestamp", curDatetime); + .withDetail("timestamp", curDatetime); } else { builder.down() .withDetail("app", APP) diff --git a/src/main/java/com/mesalab/qgw/monitor/HbaseHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/HbaseHealthIndicator.java index 43ff4ccd..5cfba23b 100644 --- a/src/main/java/com/mesalab/qgw/monitor/HbaseHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/HbaseHealthIndicator.java @@ -39,14 +39,11 @@ public class HbaseHealthIndicator extends AbstractHealthIndicator { HbaseDialect.PROPERTIES.setProperty("hbase.rpc.timeout", hBaseAPISource.getRpcTimeout()); HbaseDialect.PROPERTIES.setProperty("hbase.client.scanner.timeout.period", hBaseAPISource.getRpcTimeout()); HbaseDialect.PROPERTIES.setProperty("phoenix.query.timeoutMs", hBaseAPISource.getRpcTimeout()); - Connection conn = null; - PreparedStatement ps = null; - try { - String url = "jdbc:phoenix:".concat(hBaseAPISource.getZookeeperQuorum()).concat(":").concat(hBaseAPISource.getZookeeperZnodeParent()); - conn = DriverManager.getConnection(url, HbaseDialect.PROPERTIES); - String sql = "select 1"; - ps = conn.prepareStatement(sql); - PhoenixResultSet resultSet = (PhoenixResultSet) ps.executeQuery(); + String url = "jdbc:phoenix:".concat(hBaseAPISource.getZookeeperQuorum()).concat(":").concat(hBaseAPISource.getZookeeperZnodeParent()); + String sql = "select 1"; + try (Connection conn = DriverManager.getConnection(url, HbaseDialect.PROPERTIES); + PreparedStatement ps = conn.prepareStatement(sql); + ResultSet resultSet = ps.executeQuery()) { if (resultSet.next()) { builder.up() .withDetail("app", APP) @@ -61,10 +58,6 @@ public class HbaseHealthIndicator extends AbstractHealthIndicator { builder.down() .withDetail("app", APP) .withDetail("message", e.getMessage()); - } finally { - if (conn != null) { - conn.close(); - } } } -} +}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/qgw/monitor/HosHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/HosHealthIndicator.java index 70924044..65c3c379 100644 --- a/src/main/java/com/mesalab/qgw/monitor/HosHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/HosHealthIndicator.java @@ -1,7 +1,7 @@ package com.mesalab.qgw.monitor; import cn.hutool.core.util.StrUtil; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.qgw.service.impl.HttpClientService; import com.mesalab.services.configuration.HosConfig; import org.springframework.beans.factory.annotation.Autowired; @@ -33,7 +33,7 @@ public class HosHealthIndicator extends AbstractHealthIndicator { Map<String, String> headers = new HashMap<>(); headers.put("token", hosConfig.getToken()); Map<String, String> resultMap = httpClientService.httpGet(url, headers, 3000); - if (Integer.parseInt(resultMap.get("status")) == ResultStatusEnum.SUCCESS.getCode()) { + if (Integer.parseInt(resultMap.get("status")) == HttpStatusCodeEnum.SUCCESS.getCode()) { builder.up() .withDetail("app", APP) .withDetail("url", url) diff --git a/src/main/java/com/mesalab/qgw/monitor/JVMHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/JVMHealthIndicator.java index d9f81141..11152db4 100644 --- a/src/main/java/com/mesalab/qgw/monitor/JVMHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/JVMHealthIndicator.java @@ -21,6 +21,6 @@ public class JVMHealthIndicator extends AbstractHealthIndicator { builder.up() .withDetail("app", "JVM") .withDetail("timeZone", TimeZone.getDefault().getID()) - .withDetail("curTimestamp", DateUtils.getCurrentDate("yyyy-MM-dd'T'HH:mm:ss:SSS'Z'")); + .withDetail("timestamp", DateUtils.getCurrentDate("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); } } diff --git a/src/main/java/com/mesalab/qgw/monitor/JobAdminHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/JobAdminHealthIndicator.java index 73925236..e99738c4 100644 --- a/src/main/java/com/mesalab/qgw/monitor/JobAdminHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/JobAdminHealthIndicator.java @@ -1,7 +1,7 @@ package com.mesalab.qgw.monitor; import cn.hutool.core.util.StrUtil; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.qgw.model.basic.JobAdminHttpSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.health.AbstractHealthIndicator; @@ -45,10 +45,10 @@ public class JobAdminHealthIndicator extends AbstractHealthIndicator { try { ResponseEntity responseEntity = restTemplate.postForEntity(jobAdminHttpSource.getUrl() + "/login?", request, Map.class); - if (responseEntity.getStatusCodeValue() == ResultStatusEnum.SUCCESS.getCode()) { + if (responseEntity.getStatusCodeValue() == HttpStatusCodeEnum.SUCCESS.getCode()) { Map body = (HashMap) responseEntity.getBody(); int code = Integer.parseInt(String.valueOf(body.get("code"))); - if (code == ResultStatusEnum.SUCCESS.getCode()) { + if (code == HttpStatusCodeEnum.SUCCESS.getCode()) { builder.up() .withDetail("app", APP) .withDetail("url", jobAdminHttpSource.getUrl()) diff --git a/src/main/java/com/mesalab/qgw/monitor/MariaDBHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/MariaDBHealthIndicator.java index da4fb023..de866fc7 100644 --- a/src/main/java/com/mesalab/qgw/monitor/MariaDBHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/MariaDBHealthIndicator.java @@ -4,6 +4,7 @@ import cn.hutool.core.util.StrUtil; import com.alibaba.nacos.api.config.annotation.NacosValue; import com.google.common.base.Splitter; import com.jfinal.plugin.activerecord.Db; +import lombok.Data; import org.springframework.boot.actuate.health.AbstractHealthIndicator; import org.springframework.boot.actuate.health.Health; import org.springframework.stereotype.Component; @@ -34,17 +35,19 @@ public class MariaDBHealthIndicator extends AbstractHealthIndicator { return; } try { - List<Object> query = Db.query("SELECT CURRENT_TIMESTAMP AS curTimestamp"); + List<Object> query = Db.query("SELECT CURRENT_TIMESTAMP AS timestamp,version() AS version"); if (!CollectionUtils.isEmpty(query)) { String substring = url.substring(url.indexOf("?") + 1); Map<String, String> map = Splitter.on("&").withKeyValueSeparator("=").split(substring); + Object[] row = (Object[]) query.get(0); String timeZone = map.get("serverTimezone"); builder.up() .withDetail("app", APP) .withDetail("url", url) .withDetail("message", "ok") + .withDetail("version",StrUtil.subBefore(String.valueOf(row[1]), "-", false)) .withDetail("timeZone", timeZone) - .withDetail("curTimestamp", query.get(0)); + .withDetail("timestamp", row[0]); } else { builder.down() diff --git a/src/main/java/com/mesalab/qgw/monitor/NacosHealthIndicator.java b/src/main/java/com/mesalab/qgw/monitor/NacosHealthIndicator.java index b501a2e3..7666ebd9 100644 --- a/src/main/java/com/mesalab/qgw/monitor/NacosHealthIndicator.java +++ b/src/main/java/com/mesalab/qgw/monitor/NacosHealthIndicator.java @@ -4,7 +4,7 @@ import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.URLUtil; import cn.hutool.json.JSONUtil; import com.jayway.jsonpath.JsonPath; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.nacos.NacosConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.health.AbstractHealthIndicator; @@ -39,13 +39,15 @@ public class NacosHealthIndicator extends AbstractHealthIndicator { return; } try { - ResponseEntity responseEntity = restTemplate.getForEntity(URLUtil.normalize(nacosConfig.getServerAddr() + "/nacos/actuator/health"), Map.class); - if (responseEntity.getStatusCodeValue() == ResultStatusEnum.SUCCESS.getCode() + ResponseEntity<Map> resMap = restTemplate.getForEntity(URLUtil.normalize(nacosConfig.getServerAddr() + "/nacos/v1/console/server/state"), Map.class); + if (resMap.getStatusCodeValue() == HttpStatusCodeEnum.SUCCESS.getCode() + && responseEntity.getStatusCodeValue() == HttpStatusCodeEnum.SUCCESS.getCode() && "UP".equalsIgnoreCase(JsonPath.read(JSONUtil.toJsonStr(responseEntity.getBody()), "$.status"))) { builder.up() .withDetail("app", APP) .withDetail("url", nacosConfig.getServerAddr()) + .withDetail("version",resMap.getBody().get("version")) .withDetail("message", "ok"); } else { builder.down() diff --git a/src/main/java/com/mesalab/qgw/service/DslService.java b/src/main/java/com/mesalab/qgw/service/DSLService.java index 91d7333a..66ca4baa 100644 --- a/src/main/java/com/mesalab/qgw/service/DslService.java +++ b/src/main/java/com/mesalab/qgw/service/DSLService.java @@ -1,8 +1,8 @@ package com.mesalab.qgw.service; import com.mesalab.common.entity.BaseResult; -import com.mesalab.network.model.protocol.ProtocolTree; -import com.mesalab.qgw.model.basic.DSLProfile; +import com.mesalab.qgw.model.dsl.LiveChartProtocol; +import com.mesalab.qgw.model.basic.DSLQueryContext; import java.util.List; import java.util.Map; @@ -15,7 +15,17 @@ import java.util.Map; * @Date 2023/11/27 10:36 * @Author wWei */ -public interface DslService { +public interface DSLService { + + /** + * Desc: exec DSL + * + * @param dslProfile + * @return {@link BaseResult} + * @created by wWei + * @date 2023/12/11 15:54 + */ + BaseResult execDsl(DSLQueryContext dslProfile, boolean isDryRun); /** * Desc: Application and Protocol Summary @@ -25,7 +35,7 @@ public interface DslService { * @created by wWei * @date 2023/11/27 10:40 */ - BaseResult appAndProtocolSummary(DSLProfile dslProfile); + BaseResult appAndProtocolSummary(DSLQueryContext dslProfile, boolean isDryRun); /** * Desc: application and protocol tree composition @@ -35,7 +45,7 @@ public interface DslService { * @created by wWei * @date 2023/11/28 10:38 */ - BaseResult appAndProtocolTreeComposition(DSLProfile dslProfile); + BaseResult appAndProtocolTreeComposition(DSLQueryContext dslProfile, boolean isDryRun); /** * Desc: application and protocol tree throughput @@ -45,7 +55,7 @@ public interface DslService { * @created by wWei * @date 2023/11/28 10:38 */ - BaseResult applicationAndProtocolTreeThroughput(DSLProfile dslProfile); + BaseResult applicationAndProtocolTreeThroughput(DSLQueryContext dslProfile, boolean isDryRun); /** * Desc: top apps @@ -55,7 +65,7 @@ public interface DslService { * @created by wWei * @date 2023/11/28 10:39 */ - BaseResult applicationAndProtocolTopApp(DSLProfile dslProfile); + BaseResult applicationAndProtocolTopApp(DSLQueryContext dslProfile, boolean isDryRun); /** * Desc: app-related internal ips @@ -65,7 +75,7 @@ public interface DslService { * @created by wWei * @date 2023/11/28 10:40 */ - BaseResult applicationAndProtocolAppRelatedInternalIps(DSLProfile dslProfile); + BaseResult applicationAndProtocolAppRelatedInternalIps(DSLQueryContext dslProfile, boolean isDryRun); /** * Desc: app-throughput @@ -75,7 +85,7 @@ public interface DslService { * @created by wWei * @date 2023/11/28 10:40 */ - BaseResult applicationAndProtocolAppThroughput(DSLProfile dslProfile); + BaseResult applicationAndProtocolAppThroughput(DSLQueryContext dslProfile, boolean isDryRun); /** * Desc: app summary @@ -85,30 +95,30 @@ public interface DslService { * @created by wWei * @date 2023/11/28 10:41 */ - BaseResult applicationAndProtocolAppSummary(DSLProfile dslProfile); + BaseResult applicationAndProtocolAppSummary(DSLQueryContext dslProfile, boolean isDryRun); /** - * Desc: Get Subscriber ID Relate Client IPs + * Desc: IP Learning （FQDN-IP） * * @param dslProfile * @return {@link BaseResult} * @created by wWei - * @date 2023/11/30 11:31 + * @date 2023/12/1 11:04 */ - BaseResult realTimeDataAnalyticsSubscriberIdRelateIp(DSLProfile dslProfile); + BaseResult ipLearningFqdnRelateIp(DSLQueryContext dslProfile, boolean isDryRun); /** - * Desc: GTP-C Mobile Identities AND APNs Relate TEID + * Desc: IP Learning Active IP * * @param dslProfile * @return {@link BaseResult} * @created by wWei - * @date 2023/11/30 11:35 + * @date 2023/12/1 11:04 */ - BaseResult realTimeDataAnalyticsMobileIdentityRelateTeid(DSLProfile dslProfile); + BaseResult ipLearningActiveIp(DSLQueryContext dslProfile, boolean isDryRun); - List<ProtocolTree> buildFlatStructure(List<Map> protocolData); + List<LiveChartProtocol> buildFlatStructure(List<Map> protocols); - List<ProtocolTree> buildHierarchicalStructure(List<ProtocolTree> nodes); + List<LiveChartProtocol> buildHierarchicalStructure(List<LiveChartProtocol> nodes); } diff --git a/src/main/java/com/mesalab/qgw/service/MetadataService.java b/src/main/java/com/mesalab/qgw/service/DatabaseService.java index 3e235592..63de4cb2 100644 --- a/src/main/java/com/mesalab/qgw/service/MetadataService.java +++ b/src/main/java/com/mesalab/qgw/service/DatabaseService.java @@ -2,11 +2,14 @@ package com.mesalab.qgw.service; import com.mesalab.common.entity.BaseResult; -import org.apache.avro.Schema; +import com.mesalab.qgw.model.job.EncryptionInfo; +import com.mesalab.qgw.model.job.StorageDeletionInfo; + +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -public interface MetadataService { +public interface DatabaseService { /** * 查询schema信息 @@ -17,6 +20,8 @@ public interface MetadataService { */ Map getSchemaInfo(String type, String name, boolean displayStorageSize); + Map<String, List<Object>> getSchemaDataDict(String name); + /** * 查询字典信息 * @@ -57,7 +62,7 @@ public interface MetadataService { * @param tableName * @return */ - String getDBTypeByTableName(String tableName); + String getDBEngineByTableName(String tableName); /** @@ -80,12 +85,11 @@ public interface MetadataService { * Desc: 通过表名获取Schema * * @param name - * @return {@link Schema} + * @return {@link } * @created by wWei * @date 2021/7/27 2:41 下午 */ - Schema getSchemaByName(String name); - + LinkedHashMap<String, Object> getSchemaByName(String name); /** @@ -97,5 +101,57 @@ public interface MetadataService { */ BaseResult updateSchema(String name, Map<String, Object> schemaMap); + /** + * 系统存储配额 + * + * @return + */ + BaseResult getStorageQuota(); + + /** + * 获取每天日志存储变化 + * + * @param searchStartTime + * @param searchEndTime + * @return + */ + BaseResult dailyTrendOfStorage(String searchStartTime, String searchEndTime); + + /** + * 数据配额设置： + * 1. 调度任务 + * 2. schema TTL设置 + * + * @param list + * @return + */ + BaseResult deleteStorage(List<StorageDeletionInfo> list); + + /** + * 数据配额设置状态 + * + * @param logType + * @return + */ + BaseResult getDeleteStorageStatus(String logType); + + /** + * 获取日志储存配额利用率 + * + * @return + */ + BaseResult getStorageUsageStatus(); + + /** + * 自定义查询ID: 由 Catalog（数据库类型): resultID+query 组成。 + * + * @param resultId + * @param query + * @return {@link String} + * @created by wWei + * @date 2021/1/7 6:48 下午 + */ + String getCustomQueryId(String resultId, String query); + } diff --git a/src/main/java/com/mesalab/qgw/service/DatasetService.java b/src/main/java/com/mesalab/qgw/service/DatasetService.java new file mode 100644 index 00000000..80666408 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/DatasetService.java @@ -0,0 +1,57 @@ +package com.mesalab.qgw.service; + +import com.google.common.collect.Lists; +import com.mesalab.common.entity.BaseResult; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 数据集市管理服务 + * + * @Classname DatasetService + */ +public interface DatasetService +{ + /** + * Desc: 获取动态变量 + * + * @param + * @return {@link List<LinkedHashMap>} + */ + List<LinkedHashMap> getVariable(); + + /** + * Desc: 获取单个SQL模版 + * + * @param datasetId + * @return {@link Map<String, Object>} + */ + Map<String, Object> getDataset(String datasetId); + + /** + * Desc: 批量获取SQL模版 + * + * @param + * @return {@link List<Map<String, String>>} + */ + List<Map<String, Object>> getDatasets(List<String> ids, String category, String backendEngine); + + /** + * Desc: 结果预览 + * + * @param datasetId + * @return {@link BaseResult} + */ + BaseResult getPreview(String datasetId); + + /** + * Desc: 获取执行SQL + * + * @param + * @return + */ + String buildExecSQL(List<LinkedHashMap> variables, String sql); + +} diff --git a/src/main/java/com/mesalab/qgw/service/DiagnosisService.java b/src/main/java/com/mesalab/qgw/service/DiagnosisService.java index fd64a76a..7df569b5 100644 --- a/src/main/java/com/mesalab/qgw/service/DiagnosisService.java +++ b/src/main/java/com/mesalab/qgw/service/DiagnosisService.java @@ -7,24 +7,8 @@ public interface DiagnosisService { BaseResult validateSchema(); - BaseResult validateMetadata(); - - BaseResult runPocSQL(boolean isSaveResult, String option, String category); - BaseResult getPocSQL(String dialect, Integer queryNo); - BaseResult getMessageInfo(String param); JSONObject getVersionInfo(); - - - /** - * Desc: 一致性校验 - * - * @param - * @return {@link BaseResult} - * @created by wWei - * @date 2022/6/5 9:24 下午 - */ - BaseResult consistencyCheck(); } diff --git a/src/main/java/com/mesalab/qgw/service/HosService.java b/src/main/java/com/mesalab/qgw/service/HosService.java new file mode 100644 index 00000000..eb191901 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/HosService.java @@ -0,0 +1,20 @@ +package com.mesalab.qgw.service; + +import com.mesalab.common.entity.BaseResult; +import org.springframework.web.multipart.MultipartFile; + +import java.io.InputStream; +import java.util.List; +import java.util.Map; + + +public interface HosService { + + BaseResult getFileList(String bucketName, String prefix, Map<String, String> udfMetaParam); + + BaseResult uploadFile(String bucketName, String fileName, InputStream inputStream, Map<String, String> headers); + + BaseResult deleteFileList(String bucketName, List<String> collect); + + BaseResult getFile(String bucketName, String fileName); +} diff --git a/src/main/java/com/mesalab/qgw/service/PacketCombineDslService.java b/src/main/java/com/mesalab/qgw/service/PacketCombineDslService.java new file mode 100644 index 00000000..15c1b198 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/PacketCombineDslService.java @@ -0,0 +1,26 @@ +package com.mesalab.qgw.service; + +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; + +import java.io.InputStream; +import java.util.List; +import java.util.Map; + +/** + * packet combine service + * + * @Classname PacketCombineDslService + * @Date 2024/4/19 15:29 + * @Author wWei + */ +public interface PacketCombineDslService { + + BaseResult run(DSLQueryRequestParam dslQueryRequestParam); + + List<Map<String, Object>> getPackets(String sql); + + byte[] packetCombine(List<Map<String, Object>> records); + + Object uploadFile(String fileName, InputStream inputStream); +} diff --git a/src/main/java/com/mesalab/qgw/service/QueryJobService.java b/src/main/java/com/mesalab/qgw/service/QueryJobService.java new file mode 100644 index 00000000..ecb99f61 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/QueryJobService.java @@ -0,0 +1,66 @@ +package com.mesalab.qgw.service; + +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import com.mesalab.qgw.model.basic.SqlQueryRequestParam; + +import java.util.List; + +/** + * + * @Date 2023/12/6 18:09 + * @Author wWei + */ +public interface QueryJobService { + + /** + * Create an SQL saved query and return the query job id. It will storage the query job and results in the database. + * The query job will be executed in the background through the SAVED-QUERY-SCHEDULER. + * @param sqlQueryRequestParam + * @return {@link BaseResult} + * @created by wWei + * @date 2023/12/11 11:02 + */ + BaseResult createSQLSavedQuery(SqlQueryRequestParam sqlQueryRequestParam); + + /** + * Create an SQL ad-hoc query and return the query job id. The query will be executed immediately. + * It Support three modes of execution: NORMAL, BLOCKING, and ONESHOT. + * NORMAL: The query will be executed in the asynchronous thread pool. + * BLOCKING: The query will return the jobID when the query is completed (Sync Mode). + * ONESHOT: The query will return the results in the same call (Sync Mode). Does not return the job ID. + * @param sqlQueryRequestParam + * @return {@link BaseResult} + * @created by wWei + * @date 2023/12/11 11:28 + */ + BaseResult createSQLAdHocQuery(SqlQueryRequestParam sqlQueryRequestParam); + + /** + * Create a DSL ad-hoc query and return the query job id. The query will be executed immediately. + * It Support three modes of execution: NORMAL, BLOCKING, and ONESHOT[Recommend]. + * @param dslQueryRequestParam + * @return {@link BaseResult} + * @return + */ + BaseResult createDSLAdHocQuery(DSLQueryRequestParam dslQueryRequestParam); + + BaseResult getAdHocQueryResultById(String id); + + BaseResult getSavedQueryResultById(String id); + + BaseResult getAdHocQueryStatusById(String id); + + BaseResult getSavedQueryStatusById(String id); + + + BaseResult getSavedQueryResult(List<String> ids); + + BaseResult getAdHocQueryResult(List<String> ids); + + BaseResult getSavedQueryStatus(List<String> ids); + + BaseResult getAdHocQueryStatus(List<String> ids); + + BaseResult deleteSavedQueryById(String id); +} diff --git a/src/main/java/com/mesalab/qgw/service/QueryService.java b/src/main/java/com/mesalab/qgw/service/QueryService.java deleted file mode 100644 index 63e90323..00000000 --- a/src/main/java/com/mesalab/qgw/service/QueryService.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.common.entity.BaseResult; -import com.mesalab.qgw.model.basic.QueryProfile; - -public interface QueryService { - - BaseResult executeQuery(QueryProfile param); - -}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/qgw/service/RewriteTable.java b/src/main/java/com/mesalab/qgw/service/RewriteTable.java index 48d08beb..2a91b570 100644 --- a/src/main/java/com/mesalab/qgw/service/RewriteTable.java +++ b/src/main/java/com/mesalab/qgw/service/RewriteTable.java @@ -1,7 +1,6 @@ package com.mesalab.qgw.service; import com.mesalab.common.utils.SpringContextUtil; -import com.mesalab.qgw.service.MetadataService; import com.geedgenetworks.utils.StringUtil; import net.sf.jsqlparser.expression.Alias; import net.sf.jsqlparser.expression.ExpressionVisitor; @@ -18,7 +17,7 @@ import net.sf.jsqlparser.util.deparser.SelectDeParser; */ public class RewriteTable extends SelectDeParser { - private static MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("metadataService"); + private static DatabaseService databaseService = (DatabaseService) SpringContextUtil.getBean("databaseService"); public RewriteTable(ExpressionVisitor expressionVisitor, StringBuilder buffer) { super(expressionVisitor, buffer); @@ -27,7 +26,7 @@ public class RewriteTable extends SelectDeParser { @Override public void visit(Table table) { if (StringUtil.isBlank(table.getSchemaName())) { - table.setSchemaName(metadataService.getDBNameByTableName(table.getName())); + table.setSchemaName(databaseService.getDBNameByTableName(table.getName())); } if (((table.getAlias() == null)) || (StringUtil.isBlank(table.getAlias().getName()))) { Alias alias = new Alias(table.getName(), true); diff --git a/src/main/java/com/mesalab/qgw/service/SQLSyncQueryService.java b/src/main/java/com/mesalab/qgw/service/SQLSyncQueryService.java new file mode 100644 index 00000000..c9287a19 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/SQLSyncQueryService.java @@ -0,0 +1,10 @@ +package com.mesalab.qgw.service; + +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.model.basic.SQLQueryContext; + +public interface SQLSyncQueryService { + + BaseResult executeQuery(SQLQueryContext param); + +}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/qgw/service/SystemService.java b/src/main/java/com/mesalab/qgw/service/SystemService.java deleted file mode 100644 index 8cea8433..00000000 --- a/src/main/java/com/mesalab/qgw/service/SystemService.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.common.entity.BaseResult; -import com.mesalab.qgw.model.job.EncryptionInfo; -import com.mesalab.qgw.model.job.StorageDeletionInfo; - -import java.util.List; - -public interface SystemService { - - /** - * 系统存储配额 - * @return - */ - BaseResult getStorageQuota(); - - /** - * 获取每天日志存储变化 - * @param searchStartTime - * @param searchEndTime - * @return - */ - BaseResult dailyTrendOfStorage(String searchStartTime, String searchEndTime); - - /** - * 数据配额设置： - * 1. 调度任务 - * 2. schema TTL设置 - * @param list - * @return - */ - BaseResult deleteStorage(List<StorageDeletionInfo> list); - - /** - * 数据配额设置状态 - * @param logType - * @return - */ - BaseResult getDeleteStorageStatus(String logType); - - - /** - * 自定义查询ID: 由 Catalog（数据库类型): resultID+query 组成。 - * - * @param resultId - * @param query - * @return {@link String} - * @created by wWei - * @date 2021/1/7 6:48 下午 - */ - String getCustomQueryId(String resultId, String query); - - /** - * Desc: 通过queryId查询SQL任务执行状态 - * - * @param param - * @return {@link BaseResult} - * @created by wWei - * @date 2021/1/7 6:49 下午 - */ - BaseResult getCiphertext(EncryptionInfo param); - - -} diff --git a/src/main/java/com/mesalab/qgw/service/TrafficSpectrumDslService.java b/src/main/java/com/mesalab/qgw/service/TrafficSpectrumDslService.java new file mode 100644 index 00000000..e1393ddc --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/TrafficSpectrumDslService.java @@ -0,0 +1,39 @@ +package com.mesalab.qgw.service; + +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; + +import java.util.List; +import java.util.Map; + +/** + * TODO + * + * @Classname TrafficSpectrumDslService + * @Date 2024/5/10 10:13 + * @Author wWei + */ +public interface TrafficSpectrumDslService { + BaseResult run(DSLQueryRequestParam dslQueryRequestParam); + + List<Map<String, Object>> summary(DSLQueryRequestParam dslQueryRequestParam); + + List<Map<String, Object>> uniqIpsStat(DSLQueryRequestParam dslQueryRequestParam); + + List<Map<String, Object>> topServerIpAndServerDomain(DSLQueryRequestParam dslQueryRequestParam); + + List<Map<String, Object>> internalExternalDimensionMetricsStat(DSLQueryRequestParam dslQueryRequestParam); + + List<Map<String, Object>> networkThroughputTrend(DSLQueryRequestParam dslQueryRequestParam); + + Map<String, Object> getInternalExternalBipartiteGraph(Map<String, Object> data, Integer clientIPAppResultNum); + + Map<String, Object> mergeInternalExternalBipartiteGraph(Map<String, Object> data1, Map<String, Object> data2); + + List<Map<String, Object>> mergeLinks(List<Map<String, Object>> links); + + + Map<String, List<Map<String, Object>>> distinctNodes(Map<String, List<Map<String, Object>>> nodes); + + +} diff --git a/src/main/java/com/mesalab/qgw/service/TroubleshootingService.java b/src/main/java/com/mesalab/qgw/service/TroubleshootingService.java new file mode 100644 index 00000000..bce315d0 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/TroubleshootingService.java @@ -0,0 +1,16 @@ +package com.mesalab.qgw.service; + +import com.alibaba.fastjson2.JSONObject; +import com.mesalab.common.entity.BaseResult; + +public interface TroubleshootingService { + JSONObject getComponentStatus(); + + BaseResult consistencyCheck(); + + BaseResult validateMetadata(); + + BaseResult benchmarkTest(String test, boolean isSaved); + + BaseResult datesetVerification(); +} diff --git a/src/main/java/com/mesalab/qgw/service/UtilService.java b/src/main/java/com/mesalab/qgw/service/UtilService.java new file mode 100644 index 00000000..d535f4d7 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/UtilService.java @@ -0,0 +1,10 @@ +package com.mesalab.qgw.service; + +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.model.job.EncryptionInfo; + +public interface UtilService { + BaseResult getSQLSyntaxTree(String sql); + + BaseResult getCiphertext(EncryptionInfo param); +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/DatabaseServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/DatabaseServiceImpl.java new file mode 100644 index 00000000..9a00e7cc --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/DatabaseServiceImpl.java @@ -0,0 +1,1197 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.NumberUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.crypto.digest.DigestUtil; +import cn.hutool.json.JSONException; +import cn.hutool.json.JSONUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONWriter; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.config.listener.AbstractListener; +import com.alibaba.nacos.api.exception.NacosException; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.Encodes; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.jayway.jsonpath.JsonPath; +import com.jfinal.plugin.activerecord.Db; +import com.jfinal.plugin.activerecord.Record; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.enums.*; +import com.mesalab.common.utils.sqlparser.SQLHelper; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.common.nacos.NacosConfig; +import com.mesalab.common.nacos.NacosConst; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.HttpConfig; +import com.mesalab.qgw.model.basic.JobAdminHttpSource; +import com.mesalab.qgw.model.basic.SQLQueryContext; +import com.mesalab.qgw.model.basic.ClickHouseHttpSource; +import com.mesalab.qgw.model.job.ExecutorParam; +import com.mesalab.qgw.model.job.StorageDeletionInfo; +import com.mesalab.qgw.model.job.XxlJobInfo; +import com.mesalab.qgw.model.metadata.MetadataBean; +import com.mesalab.qgw.model.metadata.MetadataConfig; +import com.mesalab.qgw.service.DatabaseService; +import com.mesalab.qgw.service.SQLSyncQueryService; +import com.geedgenetworks.utils.StringUtil; +import com.mesalab.services.common.property.SqlPropertySourceFactory; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cglib.beans.BeanMap; +import org.springframework.context.EnvironmentAware; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.lang.reflect.AccessibleObject; +import java.lang.reflect.Field; +import java.time.Duration; +import java.util.*; + +import static com.mesalab.services.service.impl.SQLDatasetServiceImpl.QUERY_ID_SEPARATOR; + +/** + * @Date: 2021-03-11 15:48 + * @Author : liuyongqiang + * @ClassName : DatabaseServiceImpl + * @Description : 获取Nacos注册中心元数据实现类 + */ +@Service("databaseService") +@PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) +public class DatabaseServiceImpl implements DatabaseService, EnvironmentAware { + + private static final Log log = LogFactory.get(); + @Autowired + private NacosConfig nacosConfig; + @Autowired + private ConfigService systemConfigService; + @Autowired + private ConfigService pubConfigService; + @Autowired + private ClickHouseHttpSource clickHouseHttpSource; + @Autowired + private SQLSyncQueryService sqlSyncQueryService; + @Autowired + public MetadataConfig metadataConfig; + @Autowired + private JobAdminHttpSource jobAdminHttpSource; + @Autowired + private HttpClientService httpClientService; + @Autowired + HttpConfig httpConfig; + @Autowired + Environment environment; + + private Map<String, Object> schemaCache = Maps.newHashMap(); + private Map<String, Object> originalCache = Maps.newHashMap(); + private final static String KEY_REF = "$ref"; + private final static String KEY = "key"; + private final static String VALUE = "value"; + private final static String CODE = "code"; + private final static String INDEX_KEY = "index_key"; + private Environment env; + + private Map<String, String> headers = Maps.newHashMap(); + + @PostConstruct + public void init() { + addMetadataListener(); + log.info("Initializing Schema Tables Complete"); + } + + @Override + public Map<String, Object> getSchemaInfo(String type, String name, boolean displayStorageSize) { + if (MetadataType.TABLES.getValue().equals(type)) { + return loadTables(name); + } else if (MetadataType.FIELDS.getValue().equals(type)) { + return loadFields(name, displayStorageSize); + } + throw new QGWBusinessException(HttpStatusCodeEnum.NOT_FOUND.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_URL_NOT_FOUND)); + } + + @Override + public Map<String, List<Object>> getSchemaDataDict(String tableName) { + Map<String, Object> schema = getSchemaInfo(MetadataType.FIELDS.getValue(), tableName, false); + Map<String, List<Object>> schemaDataDict = Maps.newHashMap(); + if (schema != null) { + List<Map<String, Object>> read = JsonPath.read(schema, "$.fields[?(@.doc.data != null && @.doc.data.size() > 0)]"); + for (int i = 0; i < read.size(); i++) { + Map<String, Object> map = read.get(i); + String name = String.valueOf(map.get("name")); + List<Object> codes = JsonPath.read(map, "$.doc.data[*].code"); + if (codes == null || codes.isEmpty()) { + continue; + } + schemaDataDict.put(name, codes); + } + } + return schemaDataDict; + } + + @Override + public String getPartitionKey(String tableName) { + String partitionKey = StringUtil.EMPTY; + LinkedHashMap<String, Object> schemaMap = getSchemaByName(tableName); + if (schemaMap.containsKey("doc")) { + LinkedHashMap<String, Object> doc = (LinkedHashMap<String, Object>) schemaMap.get("doc"); + Object partition_key = doc.get("partition_key"); + partitionKey = StringUtil.isEmpty(partition_key) ? StringUtil.EMPTY : partition_key.toString(); + } + return partitionKey; + } + + @Override + public List<String> getIndexKey(String tableName) { + Map schemaJsonMap = loadFields(tableName, false); + if (StringUtil.isEmpty(schemaJsonMap) || StringUtil.isEmpty(schemaJsonMap.get("doc"))) { + return Lists.newArrayList(); + } + LinkedHashMap docMap = (LinkedHashMap) schemaJsonMap.get("doc"); + Object indexKey = docMap.get(INDEX_KEY); + return StringUtil.isEmpty(indexKey) ? Lists.newArrayList() : (List<String>) indexKey; + } + + @Override + public String getValueByKeyInSchemaDoc(String tableName, String key) { + LinkedHashMap<String, Object> schemaMap = getSchemaByName(tableName); + if (schemaMap.containsKey("doc")) { + LinkedHashMap<String, Object> docMap = (LinkedHashMap<String, Object>) schemaMap.get("doc"); + Object value = docMap.get(key); + if (value instanceof Map) { + return JSONUtil.toJsonStr(JSONUtil.parseObj(value)); + } else { + return StringUtil.isEmpty(value) ? StringUtil.EMPTY : value.toString(); + } + } + return StringUtil.EMPTY; + } + + @Override + public String getDBEngineByTableName(String tableName) { + LinkedHashMap<String, Object> schemaMap = getSchemaByName(tableName); + String data = StringUtil.EMPTY; + if (StringUtil.isEmpty(schemaMap)) { + return data; + } + for (MetadataBean meta : metadataConfig.getMetadata()) { + if (meta.getTables().contains(tableName)) { + return getDBTypeByGroup(meta.getGroup()); + } + } + return data; + } + + private String getDBTypeByGroup(String group) { + String type; + switch (group) { + case "CLICKHOUSE_GROUP": + type = DBEngineType.CLICKHOUSE.getValue(); + break; + case "HBASE_GROUP": + type = DBEngineType.HBASE.getValue(); + break; + case "DRUID_GROUP": + type = DBEngineType.DRUID.getValue(); + break; + default: + type = StringUtil.EMPTY; + } + return type; + } + + @Override + public String getDBNameByTableName(String tableName) { + LinkedHashMap<String, Object> schemaMap = getSchemaByName(tableName); + return StringUtil.isNotEmpty(schemaMap) ? String.valueOf(schemaMap.get("namespace")) : clickHouseHttpSource.getDbName(); + } + + @Override + public List<String> getAllTable() { + List<String> list = new ArrayList<>(); + metadataConfig.getMetadata().forEach(o -> { + if (StringUtil.isNotEmpty(o.getTables())) { + list.addAll(o.getTables()); + } + }); + return list; + } + + @Override + public LinkedHashMap<String, Object> getSchemaByName(String tableName) { + if (!getAllTable().contains(tableName)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), + CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), + tableName + ":" + QGWMessageConst.TABLE_NOT_EXIST)); + } + LinkedHashMap<String, Object> map; + if (!StrUtil.isBlankIfStr(map = (LinkedHashMap<String, Object>) schemaCache.get(tableName))) { + return map; + } + try { + String content = systemConfigService.getConfig(tableName.concat(NacosConst.JSON_SUFFIX), nacosConfig.getGroup(), 3000); + if (StrUtil.isBlankIfStr(content)) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), tableName + ":" + QGWMessageConst.QUERY_SCHEMA_ERROR)); + } + schemaCache.put(tableName, map = parseSchema(content)); + systemConfigService.addListener(tableName.concat(NacosConst.JSON_SUFFIX), nacosConfig.getGroup(), new AbstractListener() { + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo Schema Fields {}", configInfo); + schemaCache.remove(tableName); + schemaCache.put(tableName, parseSchema(configInfo)); + } + }); + } catch (NacosException e) { + log.error("NacosException:{}", e); + } + return map; + } + + @Override + public BaseResult updateSchema(String name, Map<String, Object> paramMap) { + try { + String content = systemConfigService.getConfig(name.concat(".json"), nacosConfig.getGroup(), 3000); + updateSchema(name, paramMap, content); + String indexTable = getValueByKeyInSchemaDoc(name, "index_table"); + if (StringUtil.isNotBlank(indexTable)) { + List<String> list = Splitter.on(",").omitEmptyStrings().splitToList(indexTable); + for (String item : list) { + String config = systemConfigService.getConfig(item.concat(".json"), nacosConfig.getGroup(), 3000); + updateSchema(item, paramMap, config); + } + } + return BaseResultGenerator.success4Message("ok"); + } catch (NacosException e) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + e.getMessage()); + } + } + + @Override + public BaseResult getStorageQuota() { + List<Record> storageData = Db.find(env.getProperty("SYSTEM_STORAGE_QUOTA")); + List<Map> result = Lists.newArrayList(); + BaseResult storageUsageStatus = getStorageUsageStatus(); + List<Map<String, Object>> usageData = Lists.newArrayList(); + if (storageUsageStatus.isSuccess()) { + usageData = (List<Map<String, Object>>) storageUsageStatus.getData(); + } + for (LogType value : LogType.values()) { + if (value.getValue().equals(LogType.ALL.getValue())) { + continue; + } + Map<String, Object> resultItem = Maps.newHashMap(); + storageData.forEach(record -> { + Map<String, Object> columns = record.getColumns(); + if (value.getValue().equals(columns.get("type"))) { + resultItem.putAll(columns); + } + }); + + BaseResult deleteStorageStatus = getDeleteStorageStatus(value.getValue()); + if (deleteStorageStatus.isSuccess()) { + List<StorageDeletionInfo> data = (List<StorageDeletionInfo>) deleteStorageStatus.getData(); + resultItem.put("type", value.getLabel()); + resultItem.put("max_days", data.get(0).getMaxDays()); + resultItem.put("default_max_days", data.get(0).getDefaultMaxDays()); + } + usageData.forEach(o -> { + Map<String, Object> map = JSON.parseObject(String.valueOf(o.get("executorParam")), Map.class); + if (value.getValue().equals(map.get("logType"))) { + resultItem.put("max_usage", map.get("maxUsage")); + } + }); + result.add(resultItem); + + } + return BaseResultGenerator.success(result); + } + + @Override + public BaseResult dailyTrendOfStorage(String searchStartTime, String searchEndTime) { + Date currentDate = DateUtils.convertStringToDate(DateUtils.getCurrentDate(), DateUtils.YYYY_MM_DD); + + if (StringUtil.isBlank(searchStartTime) && StringUtil.isBlank(searchEndTime)) { + searchStartTime = DateUtils.getFormatDate(DateUtils.getSomeDate(currentDate, -7), DateUtils.YYYY_MM_DD_HH24_MM_SS); + searchEndTime = DateUtils.getFormatDate(currentDate, DateUtils.YYYY_MM_DD_HH24_MM_SS); + } + + if (StringUtil.isNotBlank(searchStartTime) && StringUtil.isNotBlank(searchEndTime)) { + String sql = String.format(Objects.requireNonNull(env.getProperty("SYSTEM_DAILY_TREND_OF_STORAGE")), searchStartTime, searchEndTime); + List<Record> data = Db.find(sql); + List<Map<String, Object>> result = Lists.newArrayList(); + data.forEach(record -> { + Map<String, Object> columns = record.getColumns(); + columns.put("type", LogType.getLabelByValue(String.valueOf(columns.get("type")))); + result.add(columns); + }); + return BaseResultGenerator.success(result); + } else { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.START_TIME_AND_END_TIME_NOT_NULL)); + } + } + + @Override + public BaseResult deleteStorage(List<StorageDeletionInfo> list) { + if (!logTypeValid(list)) { + return BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), "Match failed, please check log type!")); + } + + preProcessOfLogType(list); + + if (jobIsBusy(list)) { + return BaseResultGenerator.failure(HttpStatusCodeEnum.LOCKED.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), "The task is busy, please try later!")); + } + + BaseResult baseResult = null; + for (StorageDeletionInfo info : list) { + baseResult = executeDeleteStorageJob(info); + } + return baseResult; + } + + @Override + public BaseResult getDeleteStorageStatus(String logType) { + setCookie(); + List<StorageDeletionInfo> list = new ArrayList<>(); + if (StringUtil.isBlank(logType)) { + + StorageDeletionInfo trafficInfo = getStorageDeletionInfoByHandler(LogType.TRAFFIC_LOGS.getValue(), JobHandlerOption.DELETE_TRAFFIC_DATA_JOB_HANDLER.getValue()); + list.add(trafficInfo); + + StorageDeletionInfo reportInfo = getStorageDeletionInfoByHandler(LogType.METRICS.getValue(), JobHandlerOption.DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue()); + list.add(reportInfo); + + StorageDeletionInfo fileInfo = getStorageDeletionInfoByHandler(LogType.FILES.getValue(), JobHandlerOption.DELETE_FILES_JOB_HANDLER.getValue()); + list.add(fileInfo); + } else if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { + StorageDeletionInfo trafficInfo = getStorageDeletionInfoByHandler(logType, JobHandlerOption.DELETE_TRAFFIC_DATA_JOB_HANDLER.getValue()); + list.add(trafficInfo); + } else if (LogType.METRICS.getValue().equalsIgnoreCase(logType)) { + StorageDeletionInfo reportInfo = getStorageDeletionInfoByHandler(logType, JobHandlerOption.DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue()); + list.add(reportInfo); + } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { + StorageDeletionInfo reportInfo = getStorageDeletionInfoByHandler(logType, JobHandlerOption.DELETE_FILES_JOB_HANDLER.getValue()); + list.add(reportInfo); + } else { + return BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), "No matching log type: " + logType); + } + list.forEach(o -> o.setType(LogType.getLabelByValue(o.getType()))); + return BaseResultGenerator.success("ok", list); + } + + @Override + public BaseResult getStorageUsageStatus() { + setCookie(); + List<Map> trafficDate = getAllDataByHandler(JobHandlerOption.DELETE_OLD_LOG_JOB_HANDLER.getValue()); + return BaseResultGenerator.success("ok", trafficDate); + } + + @Override + public String getCustomQueryId(String resultId, String query) { + String tableName = SQLHelper.getTableName(query).get(0); + String dbType = getDBEngineByTableName(tableName); + return DigestUtil.md5Hex(dbType) + QUERY_ID_SEPARATOR + DigestUtil.md5Hex(resultId + query.trim()); + } + + private boolean updateSchema(String name, Map<String, Object> paramMap, String content) throws NacosException { + if (StringUtil.isNotEmpty(content)) { + Map<String, Object> data = JSON.parseObject(content, Map.class); + data.put("doc", rewriteDoc("ttl", data.get("doc"), paramMap.get("doc"))); + List<Map<String, Object>> fields = (List<Map<String, Object>>) data.get("fields"); + List<String> indexKey = getIndexKey(name); + for (Map<String, Object> field : fields) { + if (indexKey.contains(field.get("name"))) { + Map<String, Object> ttlNull = Maps.newHashMap(); + ttlNull.put("ttl", null); + field.put("doc", rewriteDoc("ttl", field.get("doc"), ttlNull)); + } else { + field.put("doc", rewriteDoc("ttl", field.get("doc"), JsonPath.read(paramMap, "$.fields[?(@.name == \"" + field.get("name") + "\")].doc"))); + } + field.put("doc", rewriteDoc("visibility", field.get("doc"), JsonPath.read(paramMap, "$.fields[?(@.name == \"" + field.get("name") + "\")].doc"))); + } + content = JSON.toJSONString(data, JSONWriter.Feature.WriteNulls); + log.info("push Schema, content is: {}", content); + return systemConfigService.publishConfig(name.concat(".json"), nacosConfig.getGroup(), JSONUtil.formatJsonStr(content)); + } else { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), QGWMessageConst.QUERY_SCHEMA_ERROR)); + } + } + + /** + * Desc: 重写[tables |field].doc + * + * @param key + * @param original + * @param param + * @return {@link Map< String, Object>} + * @created by wWei + * @date 2022/5/17 10:02 上午 + */ + private Map<String, Object> rewriteDoc(String key, Object original, Object param) { + Map<String, Object> doc = Maps.newHashMap(); + if (StringUtil.isNotEmpty(original)) { + doc = (Map<String, Object>) original; + } + Map<String, Object> paramDoc = Maps.newHashMap(); + if (param instanceof Map) { + paramDoc = (Map<String, Object>) param; + } else if (param instanceof List) { + List item = (List) param; + paramDoc = (Map<String, Object>) item.get(0); + } + if (paramDoc.containsKey(key)) { + Object value = paramDoc.get(key); + if (NumberUtil.isNumber(String.valueOf(value))) { + doc.put(key, Long.parseLong(value.toString())); + } else { + doc.put(key, value); + } + } else { + doc.remove(key); + } + return doc; + } + + private void addMetadataListener() { + try { + systemConfigService.addListener(NacosConst.META_DATA_ID, nacosConfig.getGroup(), new AbstractListener() { + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo metadata {}", configInfo); + schemaCache = Maps.newHashMap(); + } + }); + } catch (NacosException e) { + log.error("NacosException: ", e); + } + } + + private Map loadFields(String name, boolean displayStorageSize) { + LinkedHashMap<String, Object> schemaMap = getSchemaByName(name); + if (StringUtil.isEmpty(schemaMap)) { + throw new QGWBusinessException(HttpStatusCodeEnum.NOT_FOUND.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), QGWMessageConst.SCHEMA_URL_NOT_FOUND); + } + LinkedHashMap schemaDoc = (LinkedHashMap) schemaMap.get("doc"); + if (StringUtil.isEmpty(schemaDoc) && + StringUtil.isNotEmpty(schemaMap.get("doc"))) { + log.error("{} schema's doc isn't jsonString and won't convert: {}", name, schemaMap.get("doc")); + } + List<Map> data = Lists.newArrayList(); + if (displayStorageSize) { + data = getStorageSize(name); + schemaDoc = (LinkedHashMap) fillSchemaStorageSize(schemaDoc, data); + } + if (StringUtil.isNotEmpty(schemaDoc)) { + schemaMap.put("doc", schemaDoc); + } + List<Map> fields = (List<Map>) schemaMap.get("fields"); + LinkedHashMap mapDoc; + for (Map next : fields) { + mapDoc = (LinkedHashMap) next.get("doc"); + mapDoc = (LinkedHashMap) fillFieldStorageSize(data, mapDoc, next); + if (StringUtil.isNotEmpty(mapDoc)) { + next.put("doc", mapDoc); + } + if (StringUtil.isEmpty(mapDoc) && !StringUtil.isEmpty(next.get("doc"))) { + log.error("{} field's doc isn't jsonString and won't convert: {}", next.get("name"), next.get("doc")); + } + } + fillReference(schemaMap, name); + return schemaMap; + } + + private Map fillSchemaStorageSize(LinkedHashMap schemaDoc, List<Map> data) { + if (CollectionUtil.isEmpty(data)) { + return schemaDoc; + } + if (StringUtil.isEmpty(schemaDoc)) { + schemaDoc = new LinkedHashMap(); + } + long total = 0; + for (Map datum : data) { + total += Long.parseLong(String.valueOf(datum.get("bytes"))); + } + schemaDoc.put("size", total); + return schemaDoc; + } + + private Map fillFieldStorageSize(List<Map> data, LinkedHashMap mapDoc, Map next) { + if (CollectionUtil.isEmpty(data)) { + return mapDoc; + } + if (StringUtil.isEmpty(mapDoc)) { + mapDoc = new LinkedHashMap(); + } + try { + List<Object> name = JsonPath.read(data, "$.[?(@.field == \"" + next.get("name") + "\")].bytes"); + mapDoc.put("size", Long.parseLong(String.valueOf(name.get(0)))); + } catch (RuntimeException e) { + log.error("fill {} field storage size error, message is: {}", next.get("name"), e.getMessage()); + } + return mapDoc; + } + + private List<Map> getStorageSize(String name) { + try { + List<String> symbols = (List<String>) loadTables(clickHouseHttpSource.getDbName()).get("symbols"); + String sql = null; + if (symbols.contains(name)) { + sql = String.format(Objects.requireNonNull(environment.getProperty("SCHEMA_STORAGE_SIZE")), name, clickHouseHttpSource.getDbName()); + } + BaseResult baseResult = StringUtil.isBlank(sql) ? BaseResultGenerator.success(Lists.newArrayList()) : sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(sql).build()); + if (baseResult.isSuccess()) { + return (List<Map>) baseResult.getData(); + } else { + log.error("get {} schema storage size error on DB.", name); + } + } catch (RuntimeException ex) { + log.error("get {} schema storage size error on DB, message is {}", name, ex.getMessage()); + } + return Lists.newArrayList(); + } + + /** + * @param map 需操作对象 + * @param cfgName 当前文件名称 + * @return void + * @Description $ref实际引用部分赋值 + * @author wanghao + * @date 2021/9/1 17:02 + */ + + private void fillReference(Map map, String cfgName) { + for (Object mapKey : map.keySet()) { + Object keyObj = map.get(mapKey); + if (keyObj instanceof Map) { + Map mapValue = (Map) keyObj; + if (mapValue.containsKey(KEY_REF)) { + if (INDEX_KEY.equals(String.valueOf(mapKey))) { + String refValue = String.valueOf(mapValue.get(KEY_REF)); + mapValue.put(KEY_REF, refValue); + } + Object refValue = getRefValue(cfgName, mapValue); + map.put(mapKey, regexValue(refValue, mapValue)); + } else { + fillReference(mapValue, cfgName); + } + } else if (keyObj instanceof List) { + List listValue = (List) keyObj; + for (Object next : listValue) { + if (next instanceof Map) { + fillReference((Map) next, cfgName); + } + } + } + } + } + + @Override + public Object getCfg(String cfgName) { + if (originalCache.containsKey(cfgName)) { + return originalCache.get(cfgName); + } + String content = null; + try { + content = getPubCfg(cfgName); + if (!StrUtil.isBlankIfStr(content)) { + originalCache.put(cfgName, content); + return content; + } + content = systemConfigService.getConfig(cfgName, nacosConfig.getGroup(), 3000); + originalCache.put(cfgName, content); + systemConfigService.addListener(cfgName, nacosConfig.getGroup(), new AbstractListener() { + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo Schema Fields {}", configInfo); + originalCache.put(cfgName, configInfo); + schemaCache.clear(); + } + }); + } catch (NacosException e) { + log.error("NacosException:{}", e); + } + return content; + } + + private String getPubCfg(String cfgName) throws NacosException { + String content = pubConfigService.getConfig(cfgName, NacosConst.DEFAULT_GROUP, 3000); + pubConfigService.addListener(cfgName, NacosConst.DEFAULT_GROUP, new AbstractListener() { + @Override + public void receiveConfigInfo(String configInfo) { + log.info("ReceiveConfigInfo Schema Fields {}", configInfo); + originalCache.put(cfgName, configInfo); + schemaCache.clear(); + } + }); + return content; + } + + private Object getRefValue(String schemaName, Map map) { + Object data = null; + String[] split = getRefStrArray(map); + String jsonPath = getJsonPath(split); + try { + Object cfg = getCfg(getFileName(split, schemaName)); + data = JsonPath.read(String.valueOf(cfg), jsonPath); + } catch (RuntimeException e) { + log.warn("read reference schema error: {}", e); + } + return data; + } + + private Object regexValue(Object obj, Map<String, String> map) { + if (map.size() <= 1) { + return obj; + } + List<Object> key = Lists.newArrayList(); + List<Object> value = Lists.newArrayList(); + try { + key.addAll(JsonPath.read(obj, map.get(KEY))); + value.addAll(JsonPath.read(obj, map.get(VALUE))); + } catch (RuntimeException ex) { + log.warn("parse reference schema error: JsonPath.read error"); + return Lists.newArrayList(); + } + if (key.size() != value.size()) { + log.warn("parse reference schema error: key and value non-correspondence"); + return Lists.newArrayList(); + } + List<Map<String, Object>> list = Lists.newArrayList(); + for (int i = 0; i < key.size(); i++) { + HashMap<String, Object> item = Maps.newLinkedHashMap(); + item.put(CODE, key.get(i)); + item.put(VALUE, value.get(i)); + list.add(item); + } + return list; + } + + private String[] getRefStrArray(Map refMap) { + String v = (String) refMap.get(KEY_REF); + return v.split("#", 2); + } + + private String getJsonPath(String[] strArray) { + return "$".concat(strArray[1].replace("/", ".")); + } + + private String getFileName(String[] strArray, String fileName) { + return StrUtil.isEmpty(strArray[0]) ? fileName.concat(NacosConst.JSON_SUFFIX) : strArray[0]; + } + + /** + * @param name + * @Description: 从Nacos配置中心获取Tables + * @Author: liuyongqiang + * @Date: 2021/3/11 18:17 + * @return: com.mesalab.common.base.BaseResult + **/ + private Map<String, Object> loadTables(String name) { + Map<String, Object> date = new LinkedHashMap<>(); + List<String> tables = new ArrayList<>(); + metadataConfig.getMetadata().forEach(o -> { + if (o.getNamespace().equalsIgnoreCase(name)) { + tables.addAll(o.getTables()); + } + } + ); + if (tables.isEmpty()) { + throw new QGWBusinessException(HttpStatusCodeEnum.NOT_FOUND.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), QGWMessageConst.SCHEMA_URL_NOT_FOUND); + } + date.put("type", "enum"); + date.put("name", name); + date.put("symbols", tables); + return date; + } + + private LinkedHashMap parseSchema(String configInfo) { + if (StrUtil.isBlank(configInfo)) return null; + LinkedHashMap schemaMap = null; + try { + schemaMap = JSON.parseObject(configInfo, LinkedHashMap.class); + Object docSchema; + if (StringUtil.isNotEmpty(docSchema = schemaMap.get("doc"))) { + schemaMap.put("doc", JSON.parseObject(JSON.toJSONString(docSchema, JSONWriter.Feature.WriteNulls), LinkedHashMap.class)); + } + Object fieldsObj; + if (StringUtil.isNotEmpty(fieldsObj = schemaMap.get("fields"))) { + List<LinkedHashMap> fields = (List<LinkedHashMap>) fieldsObj; + for (Map next : fields) { + Object docField; + if (StringUtil.isEmpty(docField = next.get("doc"))) { + continue; + } + next.put("doc", JSON.parseObject(JSON.toJSONString(docField, JSONWriter.Feature.WriteNulls), LinkedHashMap.class)); + } + } + } catch (JSONException | ClassCastException e) { + log.error("update Schema error: {}", e); + } + return schemaMap; + } + + private boolean logTypeValid(List<StorageDeletionInfo> list) { + for (StorageDeletionInfo info : list) { + String value = DatabaseServiceImpl.LogType.getValueByLabel(info.getType()); + if (StringUtil.isEmpty(value)) { + return false; + } + } + return true; + } + + /** + * @Description 涉及对ALL、LogType(value，label)处理 + * @Param list: + * @return: java.util.List<com.mesalab.qgw.model.job.StorageDeletionInfo> + * @Created by wWei + */ + private List<StorageDeletionInfo> preProcessOfLogType(List<StorageDeletionInfo> list) { + if (list.size() == 1) { + StorageDeletionInfo deletionInfo = list.get(0); + Integer maxDays = deletionInfo.getMaxDays(); + Integer defaultMaxDays = deletionInfo.getDefaultMaxDays(); + if (LogType.ALL.getValue().equalsIgnoreCase(list.get(0).getType())) { + list.clear(); + list.add(new StorageDeletionInfo(LogType.TRAFFIC_LOGS.getValue(), maxDays, defaultMaxDays)); + list.add(new StorageDeletionInfo(LogType.METRICS.getValue(), maxDays, defaultMaxDays)); + list.add(new StorageDeletionInfo(LogType.FILES.getValue(), maxDays, defaultMaxDays)); + return list; + } else if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(list.get(0).getType()) | + LogType.FILES.getValue().equalsIgnoreCase(list.get(0).getType()) | + LogType.METRICS.getLabel().equalsIgnoreCase(list.get(0).getType())) { + list.clear(); + list.add(new StorageDeletionInfo(LogType.getValueByLabel(deletionInfo.getType()), maxDays, defaultMaxDays)); + return list; + } + } + list.forEach(o -> o.setType(LogType.getValueByLabel(o.getType()))); + return list; + } + + private boolean jobIsBusy(List<StorageDeletionInfo> list) { + setCookie(); + for (StorageDeletionInfo info : list) { + + String handler = getDeletePartHandlerByLogType(info.getType()); + if (jobIsBusyByHandler(handler)) { + return true; + } + handler = getDeleteAllHandlerByLogType(info.getType()); + if (jobIsBusyByHandler(handler)) { + return true; + } + } + return false; + } + + private String getDeletePartHandlerByLogType(String logType) { + String jobHandler = StringUtil.EMPTY; + if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { + jobHandler = JobHandlerOption.DELETE_TRAFFIC_DATA_JOB_HANDLER.getValue(); + } else if (LogType.METRICS.getValue().equalsIgnoreCase(logType)) { + jobHandler = JobHandlerOption.DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue(); + } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { + jobHandler = JobHandlerOption.DELETE_FILES_JOB_HANDLER.getValue(); + } + return jobHandler; + } + + private boolean jobIsBusyByHandler(String handler) { + Map dataByHandler = getDataByHandler(handler); + String id = String.valueOf(dataByHandler.get("id")); + BaseResult result = queryJobStatusByJobId(Integer.parseInt(id)); + if (result.getStatus().equals(HttpStatusCodeEnum.LOCKED.getCode())) { + return true; + } else if (!result.getStatus().equals(HttpStatusCodeEnum.SUCCESS.getCode())) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), result.getMessage())); + } + return false; + } + + private void updateSchema(StorageDeletionInfo info, Map dataByHandler) { + Map<String, Map> previous = Maps.newHashMap(); + try { + Map schemaInfo = getSchemaInfo(MetadataType.TABLES.getValue(), clickHouseHttpSource.getDbName(), false); + Object tables = schemaInfo.get("symbols"); + if (StringUtil.isEmpty(tables)) { + return; + } + long ttl = Duration.ofDays(info.getMaxDays()).getSeconds(); + List<String> list = (List) tables; + for (String tableName : list) { + Map schemaMap = getSchemaInfo(MetadataType.FIELDS.getValue(), tableName, false); + List<Object> schemaDocTTL = JsonPath.read(schemaMap, "$.[?(@.doc.ttl != null)].doc.ttl"); + List<Object> fieldDocTTL = JsonPath.read(schemaMap, "$.fields[?(@.doc.ttl != null)].doc.ttl"); + if (schemaDocTTL.isEmpty() && fieldDocTTL.isEmpty()) { + continue; + } + previous.put(tableName, getSchemaInfo(MetadataType.FIELDS.getValue(), tableName, false)); + Object schemaDoc = schemaMap.get("doc"); + Map<String, Object> map = Maps.newHashMap(); + if (StringUtil.isNotEmpty(schemaDoc)) { + map = (Map<String, Object>) schemaDoc; + } + if (StringUtil.isNotEmpty(map.get("ttl")) && ttl < Long.parseLong(map.get("ttl").toString())) { + map.put("ttl", ttl); + schemaMap.put("doc", map); + } + List<Map<String, Object>> fields = JsonPath.read(schemaMap, "$.fields"); + List<String> indexKey = getIndexKey(tableName); + for (Map<String, Object> field : fields) { + Object doc = field.get("doc"); + if (StringUtil.isEmpty(doc)) { + doc = Maps.newHashMap(); + } + Map<String, Object> fieldDoc = (Map<String, Object>) doc; + if (indexKey.contains(field.get("name").toString())) { + fieldDoc.put("ttl", null); + field.put("doc", fieldDoc); + continue; + } + if (StringUtil.isNotEmpty(fieldDoc.get("ttl")) && ttl < Long.parseLong(fieldDoc.get("ttl").toString())) { + fieldDoc.put("ttl", ttl); + field.put("doc", fieldDoc); + } + } + updateSchema(tableName, schemaMap); + } + } catch (RuntimeException ex) { + for (String tableName : previous.keySet()) { + updateSchema(tableName, previous.get(tableName)); + } + executeManageJob("update", mapToBean(dataByHandler, XxlJobInfo.class)); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "up schema error in storage set ", ex.getMessage())); + } + } + + + /** + * 通过handler Value获取jobInfo数据 + * + * @param handlerValue + * @return + */ + private Map getDataByHandler(String handlerValue) { + StringBuilder url = new StringBuilder(jobAdminHttpSource.getUrl()). + append("/jobinfo/pageList?jobGroup=-1&triggerStatus=-1&executorHandler="). + append(handlerValue); + Map<String, String> resultPageList = httpClientService.httpGet(url.toString(), headers, httpConfig.getServerResponseTimeOut()); + if (StringUtil.isNotEmpty(resultPageList) && resultPageList.get("status").equals(String.valueOf(HttpStatusCodeEnum.SUCCESS.getCode()))) { + Map<String, Object> maps = JSON.parseObject(resultPageList.get("result"), Map.class); + if (StringUtil.isEmpty(maps)) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Get or package result exception: " + JSON.toJSONString(resultPageList))); + } + List<Map> data = (List) maps.get("data"); + if (data.size() >= 1) { + return data.get(0); + } else { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "The scheduled task has no task information matching executorHandler (" + handlerValue + ")")); + } + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Get scheduled task exception: " + JSON.toJSONString(resultPageList))); + } + + /** + * 通过Handler获取全量数据 + * + * @param handlerValue + * @return + */ + private List<Map> getAllDataByHandler(String handlerValue) { + StringBuilder url = new StringBuilder(jobAdminHttpSource.getUrl()). + append("/jobinfo/pageList?jobGroup=-1&triggerStatus=-1&executorHandler="). + append(handlerValue); + Map<String, String> resultPageList = httpClientService.httpGet(url.toString(), headers, httpConfig.getServerResponseTimeOut()); + if (StringUtil.isNotEmpty(resultPageList) && resultPageList.get("status").equals(String.valueOf(HttpStatusCodeEnum.SUCCESS.getCode()))) { + Map<String, Object> maps = JSON.parseObject(resultPageList.get("result"), Map.class); + if (StringUtil.isEmpty(maps)) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Get or package result exception: " + JSON.toJSONString(resultPageList))); + } + return (List) maps.get("data"); + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Get scheduled task exception: " + JSON.toJSONString(resultPageList))); + } + + /** + * 查询任务状态 + * + * @param jobId + * @return + */ + private BaseResult queryJobStatusByJobId(int jobId) { + BaseResult baseResult = null; + StringBuilder url = new StringBuilder(jobAdminHttpSource.getUrl()).append("/jobinfo/jobBeat?jobId=").append(jobId); + Map<String, String> resultMap = httpClientService.httpGet(url.toString(), headers, httpConfig.getServerResponseTimeOut()); + baseResult = resultEncapsulationOfJob(resultMap); + return baseResult; + } + + /** + * 执行调度任务结果封装 + * + * @param resultMap + * @return + */ + private BaseResult resultEncapsulationOfJob(Map<String, String> resultMap) { + BaseResult baseResult; + if (StringUtil.isEmpty(resultMap)) { + baseResult = BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "The service is busy, please contact the scheduling platform!")); + } else { + if (resultMap.get("status").equals(String.valueOf(HttpStatusCodeEnum.SUCCESS.getCode()))) { + Map result = JSON.parseObject(resultMap.get("result"), Map.class); + if (result.get("code").equals(HttpStatusCodeEnum.SUCCESS.getCode())) { + baseResult = BaseResultGenerator.success("ok", null); + } else if (result.get("code").equals(HttpStatusCodeEnum.LOCKED.getCode())) { + baseResult = BaseResultGenerator.failure(HttpStatusCodeEnum.LOCKED.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Clear Task is Running.")); + } else { + baseResult = BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), result.get("msg"))); + } + } else { + baseResult = BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Clear Task Failed.")); + } + } + return baseResult; + } + + private String getDeleteAllHandlerByLogType(String logType) { + String jobHandler = StringUtil.EMPTY; + if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { + jobHandler = JobHandlerOption.DELETE_ALL_TRAFFIC_DATA_JOB_HANDLER.getValue(); + } else if (LogType.METRICS.getValue().equalsIgnoreCase(logType)) { + jobHandler = JobHandlerOption.DELETE_ALL_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue(); + } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { + jobHandler = JobHandlerOption.DELETE_ALL_FILES_JOB_HANDLER.getValue(); + } + return jobHandler; + } + + /** + * 获取调度平台Cookie + * + * @return + */ + public void setCookie() { + String url = jobAdminHttpSource.getUrl(); + headers.put("Content-Type", "application/json"); + String urlParamsByMap = getUrlParamsByMap(getObjectToMap(jobAdminHttpSource)); + int socketTimeOut = httpConfig.getServerResponseTimeOut(); + Map httpPostResponseHeads = httpClientService.getHttpPostResponseHeads(url + "/login?" + urlParamsByMap, headers, socketTimeOut); + String cookie = String.valueOf(httpPostResponseHeads.get("SET-COOKIE")); + headers.put("Cookie", cookie); + } + + public static String getUrlParamsByMap(Map<String, Object> map) { + if (map == null) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (Map.Entry<String, Object> entry : map.entrySet()) { + sb.append(entry.getKey()).append("=").append(Encodes.urlEncode(String.valueOf(entry.getValue()))); + sb.append("&"); + } + String s = sb.toString(); + if (s.endsWith("&")) { + s = StringUtil.substringBeforeLast(s, "&"); + } + return s; + } + + public static Map<String, Object> getObjectToMap(Object obj) { + Map<String, Object> map = new LinkedHashMap<String, Object>(); + Class<?> clazz = obj.getClass(); + Field[] declaredFields = clazz.getDeclaredFields(); + AccessibleObject.setAccessible(declaredFields, true); + for (Field field : declaredFields) { + String fieldName = field.getName(); + Object value = null; + try { + value = field.get(obj); + } catch (IllegalAccessException e) { + log.error("Illegal Access Exception: ", e); + } + if (value == null) { + value = ""; + } + map.put(fieldName, value); + } + return map; + } + + /** + * 执行日志删除任务: 调用调度任务任务 + * + * @param info + * @return + */ + private BaseResult executeDeleteStorageJob(StorageDeletionInfo info) { + + BaseResult baseResult; + String jobHandler; + String logType = info.getType(); + ExecutorParam executorParam = new ExecutorParam(); + if (0 == info.getMaxDays()) { + jobHandler = getDeleteAllHandlerByLogType(logType); + Map dataByHandler = getDataByHandler(jobHandler); + + StorageDeletionInfo storageDeletionInfoByHandler = getStorageDeletionInfoByHandler(logType, getDeletePartHandlerByLogType(logType)); + executorParam.setMaxDays(storageDeletionInfoByHandler.getMaxDays()); + XxlJobInfo xxlJobInfo = setXxlJobInfoParam(dataByHandler, executorParam); + baseResult = executeTriggerAndUpdate(xxlJobInfo); + } else { + jobHandler = getDeletePartHandlerByLogType(info.getType()); + executorParam.setMaxDays(info.getMaxDays()); + Map oldData = getDataByHandler(jobHandler); + XxlJobInfo newData = setXxlJobInfoParam(oldData, executorParam); + baseResult = executeManageJob("update", newData); + if (baseResult.isSuccess() && LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(info.getType())) { + updateSchema(info, oldData); + } + if (baseResult.isSuccess()) { + baseResult = executeManageJob("trigger", newData); + } + } + if (!baseResult.getStatus().equals(HttpStatusCodeEnum.SUCCESS.getCode())) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), QGWMessageConst.SCHEDULED_TASK_ERROR)); + } + return baseResult; + } + + /** + * 触发执行器并更新任务 + * + * @param xxlJobInfo + * @return + */ + private BaseResult executeTriggerAndUpdate(XxlJobInfo xxlJobInfo) { + BaseResult baseResult; + BaseResult resultExecute = executeManageJob("trigger", xxlJobInfo); + if (resultExecute.getStatus().equals(HttpStatusCodeEnum.SUCCESS.getCode())) { + BaseResult resultUpdate = executeManageJob("update", xxlJobInfo); + if (resultUpdate.getStatus().equals(HttpStatusCodeEnum.SUCCESS.getCode())) { + baseResult = BaseResultGenerator.success("ok", null); + } else { + baseResult = BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Update task failed:" + xxlJobInfo.getExecutorHandler())); + } + } else { + baseResult = BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Failed to execute task:" + xxlJobInfo.getExecutorHandler())); + } + return baseResult; + } + + private StorageDeletionInfo getStorageDeletionInfoByHandler(String logType, String jobHandlerValue) { + Map trafficDate = getDataByHandler(jobHandlerValue); + XxlJobInfo xxlJobInfo = mapToBean(trafficDate, XxlJobInfo.class); + StorageDeletionInfo executorParam = JSON.parseObject(StrUtil.toUnderlineCase(xxlJobInfo.getExecutorParam()), StorageDeletionInfo.class); + executorParam.setType(logType); + return executorParam; + } + + public static <T> T mapToBean(Map<String, Object> map, Class<T> clazz) { + T bean = null; + try { + bean = clazz.newInstance(); + } catch (InstantiationException e) { + log.error("Instantiation Exception: ", e); + } catch (IllegalAccessException e) { + log.error("Illegal Access Exception: ", e); + } + BeanMap beanMap = BeanMap.create(bean); + beanMap.putAll(map); + return bean; + } + + private XxlJobInfo setXxlJobInfoParam(Map data, ExecutorParam executorParam) { + XxlJobInfo xxlJobInfo = mapToBean(data, XxlJobInfo.class); + ExecutorParam executor = JSON.parseObject(xxlJobInfo.getExecutorParam(), ExecutorParam.class); + executorParam.setDefaultMaxDays(executor.getDefaultMaxDays()); + xxlJobInfo.setExecutorParam(JSON.toJSONString(executorParam)); + return xxlJobInfo; + } + + /** + * 执行调度平台任务更新 + * + * @param executeType + * @param xxlJobInfo + * @return + */ + private BaseResult executeManageJob(String executeType, XxlJobInfo xxlJobInfo) { + BaseResult baseResult = null; + String params = getUrlParamsByMap(getObjectToMap(xxlJobInfo)); + String url = String.format("%s/jobinfo/%s/?%s", jobAdminHttpSource.getUrl(), executeType, params); + Map<String, String> resultMap = httpClientService.httpGet(url, headers, httpConfig.getServerResponseTimeOut()); + log.warn("请求调度任务" + executeType + "接口" + url); + baseResult = resultEncapsulationOfJob(resultMap); + return baseResult; + } + + @Override + public void setEnvironment(Environment environment) { + this.env = environment; + } + + @Getter + @AllArgsConstructor + enum LogType { + TRAFFIC_LOGS("Traffic Logs", "Traffic Logs"), + METRICS("Metrics", "Metrics"), + FILES("Files", "Files"), + ALL("All", "All"); + private final String value; + private final String label; + + + public static String getValueByLabel(String label) { + for (LogType enums : LogType.values()) { + if (enums.getLabel().equals(label)) { + return enums.getValue(); + } + } + return ""; + } + + public static String getLabelByValue(String value) { + for (LogType enums : LogType.values()) { + if (enums.getValue().equals(value)) { + return enums.getLabel(); + } + } + return ""; + } + } +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/DatasetServiceImp.java b/src/main/java/com/mesalab/qgw/service/impl/DatasetServiceImp.java new file mode 100644 index 00000000..b923b28e --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/DatasetServiceImp.java @@ -0,0 +1,206 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.alibaba.fastjson2.JSON; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.StringUtil; +import com.google.common.collect.Lists; +import com.jfinal.plugin.activerecord.Db; +import com.jfinal.plugin.activerecord.Record; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.enums.*; +import com.mesalab.common.nacos.NacosConst; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.*; +import com.mesalab.qgw.service.DatabaseService; +import com.mesalab.qgw.service.QueryJobService; +import com.mesalab.qgw.service.SQLSyncQueryService; +import com.mesalab.services.common.property.SqlPropertySourceFactory; +import com.mesalab.qgw.service.DatasetService; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +@Service("datasetService") +@PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) +public class DatasetServiceImp + implements DatasetService { + private static final Log log = LogFactory.get(); + private static Pattern pFieldVariable = Pattern.compile("\\$\\{(metric|dimension)_(.*?)\\}", Pattern.CASE_INSENSITIVE); + private static Pattern pLeftRightFlag = Pattern.compile("\\[\\[(.*?)\\]\\]", Pattern.CASE_INSENSITIVE); + private final static String TEMPLATE = "template"; + private final static String BACKEND_ENGINE = "backend_engine"; + + private final static String DATASET_TYPE = "type"; + @Autowired + Environment env; + @Autowired + DatabaseService databaseService; + @Autowired + SQLSyncQueryService sqlSyncQueryService; + @Autowired + private QueryJobService queryJobService; + + @Override + public List<LinkedHashMap> getVariable() { + Object codeInfo = databaseService.getCfg(NacosConst.DATASETS_VARIABLES); + if (StringUtil.isNotEmpty(codeInfo)) { + Object json = JSON.toJSON(codeInfo); + List<LinkedHashMap> variables = JSON.parseArray(json.toString(), LinkedHashMap.class); + for (LinkedHashMap variable : variables) { + String key = String.valueOf(variable.get("key")); + String def = String.valueOf(variable.get("default")); + Date currentDate = DateUtils.convertStringToDate(DateUtils.getCurrentDate(DateUtils.YYYY_MM_DD_HH24_MM_SS), DateUtils.YYYY_MM_DD_HH24_MM_SS); + if ("start_time".equals(key)&& StringUtil.isBlank(def)){ + def = DateUtils.getFormatDate(DateUtils.getSomeHour(currentDate, -1), DateUtils.YYYY_MM_DD_HH24_MM_SS); + variable.put("default", def); + } + if ("end_time".equals(key)&& StringUtil.isBlank(def)){ + def = DateUtils.getFormatDate(currentDate, DateUtils.YYYY_MM_DD_HH24_MM_SS); + variable.put("default", def); + } + } + return variables; + } + return Lists.newArrayList(); + } + + @Override + public Map<String, Object> getDataset(String datasetId) { + List<Record> list = Db.find(buildGetDatasetSQL(Lists.newArrayList(datasetId), null, null)); + if (CollectionUtils.isEmpty(list)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getMessage(),QGWMessageConst.DATASET_ID_NOT_EXIST)); + } + List<Map<String, Object>> results = formatResultData(list); + Map<String, Object> result = results.get(0); + log.info("ID is: {}, Dataset is: {}", datasetId, result); + return result; + } + + @Override + public List<Map<String, Object>> getDatasets(List<String> ids, String category, String backendEngine) { + String sql = buildGetDatasetSQL(ids, category, backendEngine); + List<Record> list = Db.find(sql); + if (CollectionUtils.isEmpty(list)) { + return Lists.newArrayList(); + } + return formatResultData(list); + } + + private static List<Map<String, Object>> formatResultData(List<Record> list) { + List<Map<String, Object>> results = new ArrayList<>(); + list.forEach(record -> { + Map<String, Object> columns = record.getColumns(); + columns.put("generated_time", Instant.ofEpochSecond(Long.parseLong(String.valueOf(columns.get("generated_time"))))); + columns.put("last_update_time", Instant.ofEpochSecond(Long.parseLong(String.valueOf(columns.get("last_update_time"))))); + results.add(record.getColumns()); + }); + return results; + } + + @Override + public BaseResult getPreview(String datasetId) { + Map<String, Object> dataset = getDataset(datasetId); + String template = buildExecSQL(getVariable(), String.valueOf(dataset.get(TEMPLATE))); + String datasetType = String.valueOf(dataset.get(DATASET_TYPE)); + if (datasetType.equalsIgnoreCase("sql")) { + SqlQueryRequestParam sqlQueryRequest = JSON.parseObject(template, SqlQueryRequestParam.class); + sqlQueryRequest.setExecutionMode(ExecutionMode.ONESHOT.getValue()); + log.info("Dataset Preview, ID is: {}, Type is: {}, Exec SQL is: {}", datasetId, "sql", sqlQueryRequest.getStatement()); + return queryJobService.createSQLAdHocQuery(sqlQueryRequest); + } else if (datasetType.equalsIgnoreCase("dsl")) { + DSLQueryRequestParam dslQueryRequest = JSON.parseObject(template, DSLQueryRequestParam.class); + dslQueryRequest.setExecutionMode(ExecutionMode.ONESHOT.getValue()); + log.info("Dataset Preview, ID is: {}, Type is: {}, DSL is: {}", datasetId, "dsl", template); + return queryJobService.createDSLAdHocQuery(dslQueryRequest); + } else { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getMessage(), "not Supported")); + } + } + + + @Override + public String buildExecSQL(List<LinkedHashMap> variables, String sql) { + Matcher matcher = pLeftRightFlag.matcher(sql); + while (matcher.find()) { + sql = processOptionalClause(sql, "[[", "]]"); + } + sql = processFieldVariable(sql); + for (LinkedHashMap linkedHashMap : variables) { + String variable = String.valueOf(linkedHashMap.get("key")); + String def = String.valueOf(linkedHashMap.get("default")); + if ("filter".equals(variable) && StringUtil.isBlank(def)) { + def = " 1 = 1"; + } + String parameter = "${".concat(variable).concat("}"); + sql = sql.replace(parameter, def); + } + return sql; + } + + private String buildGetDatasetSQL(List<String> ids, String category, String backendEngine) { + List<String> filterList = Lists.newArrayList(); + if (!ids.isEmpty()) { + filterList.add(" identifier_name IN ( '".concat(String.join("', '", ids)).concat("')")); + } + if (StrUtil.isNotBlank(category)) { + filterList.add(" category = '".concat(category.replace("'", "\\'")).concat("'")); + } + if (StrUtil.isNotBlank(backendEngine)) { + filterList.add(" backend_engine = '".concat(backendEngine.replace("'", "\\'")).concat("'")); + } + String filter = String.join(" AND ", filterList); + return String.format(Objects.requireNonNull(env.getProperty("SQL_DATASETS")), StrUtil.isNotBlank(filter) ? "WHERE ".concat(filter) : ""); + } + + private String processFieldVariable(String str) { + Matcher matcher = pFieldVariable.matcher(str); + StringBuffer sb = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(sb, matcher.group(2)); + } + matcher.appendTail(sb); + return sb.toString(); + } + + private String processOptionalClause(String str, String leftFlag, String rightFlag) { + String left = ""; + String right = ""; + String center = ""; + boolean leftMark = false; + for (int i = 0; i < str.length(); i++) { + String element = str.substring(i, i + 1); + if (leftFlag.startsWith(element) && leftFlag.equals(str.substring(i, i + leftFlag.length()))) { + left = str.substring(0, i); + leftMark = true; + continue; + } + if (leftMark) { + if (rightFlag.startsWith(element) && rightFlag.equals(str.substring(i, i + rightFlag.length()))) { + right = str.substring(i + rightFlag.length()); + center = str.substring(left.length(), str.length() - (right.length())); + break; + } + } + } + if (StrUtil.isNotEmpty(center)) { + String substring = center.substring(leftFlag.length(), center.length() - leftFlag.length()); + str = left.concat(substring).concat(right); + } + return str; + } +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/DiagnosisServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/DiagnosisServiceImpl.java index 24d15708..d6f7d20e 100644 --- a/src/main/java/com/mesalab/qgw/service/impl/DiagnosisServiceImpl.java +++ b/src/main/java/com/mesalab/qgw/service/impl/DiagnosisServiceImpl.java @@ -1,86 +1,45 @@ package com.mesalab.qgw.service.impl; -import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.file.FileReader; -import cn.hutool.core.io.file.FileWriter; -import cn.hutool.core.util.IdUtil; -import cn.hutool.core.util.NumberUtil; -import cn.hutool.core.util.StrUtil; -import cn.hutool.core.util.URLUtil; -import cn.hutool.http.HttpStatus; -import cn.hutool.json.JSONUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson2.JSON; import com.alibaba.nacos.api.config.ConfigService; import com.alibaba.nacos.api.exception.NacosException; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; -import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; -import com.jayway.jsonpath.JsonPath; -import com.jfinal.plugin.activerecord.Db; -import com.jfinal.plugin.activerecord.Record; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; import com.mesalab.common.enums.*; import com.mesalab.common.nacos.NacosConfig; -import com.mesalab.common.nacos.NacosConst; import com.mesalab.qgw.benchmark.DialectWriter; -import com.mesalab.qgw.constant.QGWMessageConst; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; +import com.mesalab.common.exception.CommonErrorCode; import com.mesalab.qgw.model.basic.*; -import com.mesalab.qgw.service.QueryService; import com.mesalab.qgw.service.DiagnosisService; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.services.common.enums.EntityQueryType; +import com.mesalab.qgw.service.DatabaseService; import com.mesalab.services.common.property.SqlPropertySourceFactory; -import com.mesalab.services.service.EntityService; import com.geedgenetworks.utils.*; -import com.mesalab.services.service.SQLDatasetService; import lombok.Data; -import org.apache.avro.Schema; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.math3.stat.StatUtils; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.EnvironmentAware; import org.springframework.context.annotation.PropertySource; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; -import javax.annotation.Resource; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; @Service("diagnosisService") @PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) -public class -DiagnosisServiceImpl implements DiagnosisService, EnvironmentAware { +public class DiagnosisServiceImpl implements DiagnosisService, EnvironmentAware { private static final Log log = LogFactory.get(); - private static final String AVRO_SUFFIX = ".avsc"; - private static final String METADATA_PREFIX = "Metadata/"; private static final String SCHEMA_PREFIX = "Schema/"; - private static final String HBASE_DBNAME2 = "tsg_galaxy"; - private static final String EXECUTE_ENGINE= "execute_engine"; - private static final String IDENTIFIER_NAME = "identifier_name"; - private static final String TEMPLATE = "template"; - - @Autowired - private QueryService queryService; @Autowired private DialectWriter dialectWriter; @@ -91,36 +50,17 @@ DiagnosisServiceImpl implements DiagnosisService, EnvironmentAware { @Autowired private HBaseAPISource hBaseAPISource; @Autowired - private MetadataService metadataService; - @Autowired - private EntityConfigSource entityConfigSourceTemp; - @Autowired - private EntityService entityService; - @Autowired - private SQLDatasetService sqlDatasetService; - + private DatabaseService databaseService; @Autowired private NacosConfig nacosConfig; @Autowired private ConfigService systemConfigService; - - @Autowired - private HttpClientService httpClientService; @Autowired HttpConfig httpConfig; - @PostConstruct - public void init() { - entityConfigSource = this.entityConfigSourceTemp; - } - private Environment env; - private static EntityConfigSource entityConfigSource; private final static String DATA_ID = "version.json"; - private static Pattern pTTL = Pattern.compile(".*toIntervalSecond\\((\\d+)\\)", Pattern.CASE_INSENSITIVE); - private static final String trafficPort = "8123"; - @Override public JSONObject getVersionInfo() { @@ -147,132 +87,24 @@ DiagnosisServiceImpl implements DiagnosisService, EnvironmentAware { Stopwatch watch = Stopwatch.createStarted(); Map<String, Object> statistics = Maps.newLinkedHashMap(); List<Object> data = Lists.newArrayList(); - Map<String, Map<String, String>> clickhouseResult = parseSchemaByDatasource(clickHouseHttpSource.getDbName(), metadataService.getAllTable()); - Map<String, Map<String, String>> druidResult = parseSchemaByDatasource(druidIoHttpSource.getDbname(), metadataService.getAllTable()); - Map<String, Map<String, String>> hbaseResult = parseSchemaByDatasource(hBaseAPISource.getDbName(), metadataService.getAllTable()); - Map<String, Map<String, String>> hbase2Result = parseSchemaByDatasource(HBASE_DBNAME2, metadataService.getAllTable()); + Map<String, Map<String, String>> clickhouseResult = parseSchemaByDatasource(clickHouseHttpSource.getDbName(), databaseService.getAllTable()); + Map<String, Map<String, String>> druidResult = parseSchemaByDatasource(druidIoHttpSource.getDbname(), databaseService.getAllTable()); + Map<String, Map<String, String>> hbaseResult = parseSchemaByDatasource(hBaseAPISource.getDbName(), databaseService.getAllTable()); data.add(clickhouseResult.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); data.add(druidResult.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); data.add(hbaseResult.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - data.add(hbase2Result.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); long executeTime = watch.elapsed(TimeUnit.MILLISECONDS); statistics.put("elapsed", executeTime); - result = BaseResultGenerator.success("Avro Schema Validation Success.", data, statistics); - result.setFormatType(QueryFormatEnum.JSON.getValue()); + result = BaseResultGenerator.success("Schema Validation Success.", data, statistics); + result.setOutputMode(OutputMode.JSON.getValue()); } catch (RuntimeException e) { - log.error("Avro Schema Validation Fail:{}", e); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - QGWErrorCode.SQL_QUERY_FEDERATION_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_QUERY_FEDERATION_EXCEPTION.getMessage(),e.getMessage())); - - } - return result; - } - - /** - * 验证schema与数据库表结构是否一致 - * - * @return - */ - @Override - public BaseResult validateMetadata() { - BaseResult result = null; - try { - Stopwatch watch = Stopwatch.createStarted(); - Map<String, Object> statistics = Maps.newLinkedHashMap(); - List<Object> data = Lists.newArrayList(); - checkCKIndexKey(clickHouseHttpSource.getDbName(), metadataService.getAllTable()); - Map<String, Map<String, String>> clickhouseResult = checkMetadataByDatasource(clickHouseHttpSource.getDbName(), metadataService.getAllTable()); - Map<String, Map<String, String>> druidResult = checkMetadataByDatasource(druidIoHttpSource.getDbname(), metadataService.getAllTable()); - Map<String, Map<String, String>> hbaseResult = checkMetadataByDatasource(hBaseAPISource.getDbName(), metadataService.getAllTable()); - Map<String, Map<String, String>> hbase2Result = checkMetadataByDatasource(HBASE_DBNAME2, metadataService.getAllTable()); - - data.add(clickhouseResult.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - data.add(druidResult.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - data.add(hbaseResult.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - data.add(hbase2Result.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - - statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); - result = BaseResultGenerator.success("Metadata Validation Success, Only Table Exist Fields detail.", data, statistics); - result.setFormatType(QueryFormatEnum.JSON.getValue()); - } catch (RuntimeException e) { - log.error("Metadata Validation Fail: {}", e.getMessage()); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), e.getMessage())); - } - return result; - } - - - @Override - public BaseResult runPocSQL(boolean isSaved, String option, String category) { - BaseResult result = null; - Map<String, Object> statistics = Maps.newLinkedHashMap(); - Map<String, Map<String, String>> mergeResult; - Map<String, Map<String, String>> engineResult; - Map<String, Map<String, String>> clickhouseResult; - Map<String, Map<String, String>> druidResult; - Map<String, Map<String, String>> hbaseResult; - try { - Map<String, Object> datasetResult = generateDatasets(category); - List<Map<String, String>> dataSetlist = (ArrayList<Map<String, String>>) datasetResult.get("list"); - List<String> druidIdList = Lists.newLinkedList(); - List<String> ckIdList = Lists.newLinkedList(); - List<String> engineIdList = Lists.newLinkedList(); - List<String> hbaseIdList = Lists.newLinkedList(); - - List<String> druidSqlList = Lists.newLinkedList(); - List<String> ckSqlList = Lists.newLinkedList(); - List<String> engineSqlList = Lists.newLinkedList(); - List<String> hbaseSqlList = Lists.newLinkedList(); - Stopwatch watch = Stopwatch.createStarted(); - if (CollectionUtil.isNotEmpty(dataSetlist)){ - for (int i = 0; i < dataSetlist.size(); i++) { - if (dataSetlist.get(i).get(EXECUTE_ENGINE).equalsIgnoreCase(DBTypeEnum.DRUID.getValue())) { - druidIdList.add(dataSetlist.get(i).get(IDENTIFIER_NAME)); - druidSqlList.add(dataSetlist.get(i).get(TEMPLATE)); - } else if (dataSetlist.get(i).get(EXECUTE_ENGINE).equalsIgnoreCase(DBTypeEnum.CLICKHOUSE.getValue())) { - ckIdList.add(dataSetlist.get(i).get(IDENTIFIER_NAME)); - ckSqlList.add(dataSetlist.get(i).get(TEMPLATE)); - } else if (dataSetlist.get(i).get(EXECUTE_ENGINE).equalsIgnoreCase(DBTypeEnum.HBASE.getValue())) { - hbaseIdList.add(dataSetlist.get(i).get(IDENTIFIER_NAME)); - hbaseSqlList.add(dataSetlist.get(i).get(TEMPLATE)); - } else { - engineIdList.add(dataSetlist.get(i).get(IDENTIFIER_NAME)); - engineSqlList.add(dataSetlist.get(i).get(TEMPLATE)); - } - } - } - - Optional.of(new File(dialectWriter.buildPocSQL(druidSqlList, DBTypeEnum.DRUID.getValue(), category))); - Optional.of(new File(dialectWriter.buildPocSQL(ckSqlList, DBTypeEnum.CLICKHOUSE.getValue(), category))); - Optional.of(new File(dialectWriter.buildPocSQL(hbaseSqlList, DBTypeEnum.HBASE.getValue(), category))); - Optional.of(new File(dialectWriter.buildPocSQL(engineSqlList, DBTypeEnum.ENGINE.getValue(), category))); + log.error("Schema Validation Fail:{}", e); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(),e.getMessage())); - clickhouseResult = buildResult(DBTypeEnum.CLICKHOUSE.getValue(), ckIdList, isSaved, option); - druidResult = buildResult(DBTypeEnum.DRUID.getValue(), druidIdList, isSaved, option); - engineResult = buildResult(DBTypeEnum.ENGINE.getValue(), engineIdList, isSaved, option); - hbaseResult = buildResult(DBTypeEnum.HBASE.getValue(), hbaseIdList, isSaved, option); - - mergeResult = Stream - .concat(clickhouseResult.entrySet().stream(), druidResult.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - mergeResult = Stream.concat(mergeResult.entrySet().stream(), engineResult.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - mergeResult = Stream.concat(mergeResult.entrySet().stream(), hbaseResult.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); - result = BaseResultGenerator.success("SQL Queries Success.", mergeResult, statistics); - result.setFormatType(QueryFormatEnum.JSON.getValue()); - } catch (RuntimeException e) { - log.error("Execute Poc SQL Fail:{}", e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), - QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getMessage(),e.getMessage())); } return result; } @@ -302,380 +134,22 @@ DiagnosisServiceImpl implements DiagnosisService, EnvironmentAware { } statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); result = BaseResultGenerator.success("Get SQL Success.", dialectResult, statistics); - result.setFormatType(QueryFormatEnum.JSON.getValue()); + result.setOutputMode(OutputMode.JSON.getValue()); } catch (RuntimeException | IOException e) { log.error("Get Poc SQL Fail:{}", e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), - ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),e.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), + CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(),e.getMessage())); } return result; } - @Override - public BaseResult consistencyCheck() { - List<Map<String, Object>> dataList = Lists.newArrayList(); - Map<String, Object> logDiff = Maps.newHashMap(); - Map tables = metadataService.getSchemaInfo(MetadataTypeEnum.TABLES.getValue(), clickHouseHttpSource.getDbName(), false); - List<String> symbols = (List<String>) tables.get("symbols"); - logDiff.put("logType", SystemServiceImpl.LogType.TRAFFIC_LOGS.getValue()); - Map<String, Map<String, Object>> changeTTL = Maps.newHashMap(); - List<String> ipPorts = getClusterAddressOfCK(); - try { - for (String ipPort : ipPorts) { - if (!changeTTL.isEmpty()) { - break; - } - if (ipPort.startsWith("127.0.0.1")) { - String url = clickHouseHttpSource.getUrl(); - ipPort = url.substring(url.indexOf("//") + 2, url.lastIndexOf(":") + 1).concat(trafficPort); - } - for (String tableName : symbols) { - Map schemaMap = metadataService.getSchemaInfo("fields", tableName, false); - Object tableTTLInSchema = getTableTTLInSchema(schemaMap); - Map<String, Object> tableDiff = getTableDiff(ipPort, tableName, tableTTLInSchema); - List fieldDiff = getFieldDiff(ipPort, tableName, schemaMap); - if (CollectionUtil.isNotEmpty(fieldDiff)) { - tableDiff.put("fields", fieldDiff); - - } - if (CollectionUtil.isNotEmpty(tableDiff)) { - changeTTL.put(tableName, tableDiff); - logDiff.put("tables", Lists.newArrayList(changeTTL)); - logDiff.put("address", Lists.newArrayList(ipPort)); - } - } - } - } catch (RuntimeException ex) { - log.error(" schema TTL check error: {}", ex); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),ex.getMessage())); - } - if (changeTTL.isEmpty()) { - return BaseResultGenerator.success("ok", dataList); - } - dataList.add(logDiff); - return BaseResultGenerator.success("Log streaming setting task not executed or failed", dataList); - } - - private Object getTableTTLInSchema(Map schemaMap) { - Object tableTTLInSchema = null; - if (StringUtil.isNotEmpty(schemaMap)) { - if (schemaMap.containsKey("doc")) { - Map<String, Object> schemaHasDoc = (Map<String, Object>) schemaMap.get("doc"); - if (schemaHasDoc.containsKey("ttl")) { - tableTTLInSchema = StringUtil.isEmpty(schemaHasDoc.get("ttl")) ? null : schemaHasDoc.get("ttl").toString(); - } - } - } - return tableTTLInSchema; - } - - private Map<String, Object> getTableDiff(String ipPort, String tableName, Object tableTTLInSchema) { - String sql = String.format(Objects.requireNonNull(env.getProperty("TABLE_TTL")), tableName, clickHouseHttpSource.getDbName()); - Map<String, String> result = executeHttpGetOfCK(ipPort, sql, clickHouseHttpSource.getSystemDBName()); - if (Integer.parseInt(result.get("status")) != ResultStatusEnum.SUCCESS.getCode()) { - log.error(" query table TTL error: {}", result.toString()); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getMessage(),result.toString())); - } - Object tableTTLInDB = null; - Map<String, Object> tableDiff = Maps.newHashMap(); - Map o = (Map) JSON.parseObject(result.get("result"), Map.class); - Map<String, Object> resultMap = (Map<String, Object>) o; - for (Map<String, Object> datum : (List<Map<String, Object>>)resultMap.get("data")) { - Matcher matcher = pTTL.matcher(String.valueOf(datum.get("table_ttl"))); - tableTTLInDB = matcher.find() ? matcher.group(1) : null; - } - if (!String.valueOf(tableTTLInDB).equals(String.valueOf(tableTTLInSchema))) { - tableDiff.put("last_ttl", StringUtil.isEmpty(tableTTLInSchema) ? null : tableTTLInSchema.toString()); - tableDiff.put("used_ttl", StringUtil.isEmpty(tableTTLInDB) ? null : tableTTLInDB.toString()); - log.warn("table ttl: {} not consistency.", tableName); - } - return tableDiff; - } - - private List<Map<String, Object>> getFieldDiff(String ipPort, String tableName, Map<String, Object> schemaMap) { - List<Map<String, Object>> fieldDiff = Lists.newArrayList(); - Map<String, String> result = executeHttpGetOfCK(ipPort, "describe " + tableName + "_local", clickHouseHttpSource.getDbName()); - if (Integer.parseInt(result.get("status")) != ResultStatusEnum.SUCCESS.getCode()) { - log.error(" query table field TTL error: {}", result.toString()); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getMessage(),result.toString())); - } - List<Map<String, Object>> fields = (List<Map<String, Object>>) schemaMap.get("fields"); - for (Map<String, Object> field : fields) { - Object read = JsonPath.read(result, "$.result"); - List<Object> fieldList = JsonPath.read(JSONUtil.toJsonStr(JSONUtil.parseObj(read)), "$.data[?(@.name == \"" + field.get("name") + "\")].ttl_expression"); - Object fieldTTLInSchema = null; - Object fieldTTLInDB; - if (StringUtil.isNotEmpty(field.get("doc"))) { - Map<String, Object> doc = (Map<String, Object>) field.get("doc"); - fieldTTLInSchema = StringUtil.isEmpty(doc.get("ttl")) ? null : doc.get("ttl"); - } - Matcher matcher = pTTL.matcher(String.valueOf(fieldList.get(0))); - fieldTTLInDB = matcher.find() ? matcher.group(1) : null; - if (String.valueOf(fieldTTLInSchema).equals(String.valueOf(fieldTTLInDB))) { - continue; - } - log.warn("field ttl: address {} {}-{} not consistency.", ipPort, tableName, field.get("name")); - Map<String, Object> ttlDiff = Maps.newHashMap(); - ttlDiff.put("last_ttl", StringUtil.isEmpty(fieldTTLInSchema) ? null : fieldTTLInSchema.toString()); - ttlDiff.put("used_ttl", StringUtil.isEmpty(fieldTTLInDB) ? null : fieldTTLInDB.toString()); - if (fieldDiff.stream().noneMatch(o -> o.containsKey(field.get("name").toString()))) { - Map<String, Object> item = Maps.newHashMap(); - item.put(field.get("name").toString(), ttlDiff); - fieldDiff.add(item); - } - } - return fieldDiff; - } - - private List<String> getClusterAddressOfCK() { - List<String> endpointList = new ArrayList<>(); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query("SELECT DISTINCT concat(host_address,':','" + trafficPort + "') as endpoint FROM clusters where cluster = 'ck_cluster'").build()); - if (baseResult.isSuccess()) { - List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); - for (Map<String, Object> datum : data) { - endpointList.add(String.valueOf(datum.get("endpoint"))); - } - } - return endpointList; - } - - private Map<String, String> executeHttpGetOfCK(String ipPort, String sql, String dbName) { - String queryURL = URLUtil.normalize("http://" + ipPort + "/?"); - StringBuilder paramBuilder = new StringBuilder("user=") - .append(clickHouseHttpSource.getRealTimeAccountUserName()).append("&") - .append("password=").append(clickHouseHttpSource.getRealTimeAccountPin()).append("&") - .append("database=").append(dbName).append("&") - .append("query=").append(sql) - .append(" FORMAT JSON;"); - List<NameValuePair> values = URLEncodedUtils.parse(paramBuilder.toString(), StandardCharsets.UTF_8); - int socketTimeOut = httpConfig.getCkRealTimeAccountSocketTimeOut(); - return httpClientService.httpGet(queryURL + URLEncodedUtils.format(values, "utf-8"), socketTimeOut); - } - /** - * - * @param param - * @Description 获取Entity推荐全网占比统计信息 - * @author wanghao - * @date 2022/7/7 15:49 - */ - @Override - public BaseResult getMessageInfo(String param) { - BaseResult baseResult = null; - Map<String, String> data = Maps.newHashMap(); - Map<String, Object> statistics = Maps.newLinkedHashMap(); - Stopwatch watch = Stopwatch.createStarted(); - String timeInterval = getTimeInterval(-24 * 60); - String datasource = "session_record"; - if (EntityQueryType.TOPSERVERIP.getType().equalsIgnoreCase(param)) { - String message = getTopServerIpMessage(timeInterval, datasource); - data.put("message", message); - } else if (EntityQueryType.TOPSNI.getType().equalsIgnoreCase(param)) { - String message = getTopSniMessage(timeInterval, datasource); - data.put("message", message); - } - statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); - baseResult = BaseResultGenerator.success("ok", data, statistics); - return baseResult; - } - - private String getTopServerIpMessage(String timeInterval, String datasource) { - String tcpSessionSql = String.format(Objects.requireNonNull(env.getProperty("ENTITY_TCP_SESSION")), datasource, timeInterval, - entityConfigSource.getTopServerIPByTCPSession()); - List<Map> tcpSessionResult = getData(queryService.executeQuery(queryBuild(tcpSessionSql))); - - String tcpClientIpsSql = String.format(Objects.requireNonNull(env.getProperty("TOP_ENTITY_TCP_UNIQ_CLIENT_IPS")), datasource, timeInterval, - datasource, timeInterval, entityConfigSource.getTopServerIPByTCPUniqClientIP()); - List<Map> tcpClientIpsResult = getData(queryService.executeQuery(queryBuild(tcpClientIpsSql))); - - String udpSessionSql = String.format(Objects.requireNonNull(env.getProperty("ENTITY_UDP_SESSION")), datasource, timeInterval, - entityConfigSource.getTopServerIPByUDPBySession()); - List<Map> udpSessionResult = getData(queryService.executeQuery(queryBuild(udpSessionSql))); - - String udpClientIpsSql = String.format(Objects.requireNonNull(env.getProperty("TOP_ENTITY_UDP_UNIQ_CLIENT_IPS")), datasource, timeInterval, - datasource, timeInterval, entityConfigSource.getTopServerIPByUDPUniqClientIP()); - List<Map> udpClientIpsResult = getData(queryService.executeQuery(queryBuild(udpClientIpsSql))); - - String tcpTotalSessionSql = String.format(Objects.requireNonNull(env.getProperty("TOTAL_ENTITY_TCP_SESSION")), datasource, timeInterval); - List<Map> tcpTotalSessionResult = getData(queryService.executeQuery(queryBuild(tcpTotalSessionSql))); - - String tcpTotalClientIpsSql = String.format(Objects.requireNonNull(env.getProperty("TOTAL_ENTITY_TCP_UNIQ_CLIENT_IPS")), datasource, timeInterval); - List<Map> tcpTotalClientIpResult = getData(queryService.executeQuery(queryBuild(tcpTotalClientIpsSql))); - - String udpTotalSessionSql = String.format(Objects.requireNonNull(env.getProperty("TOTAL_ENTITY_UDP_SESSION")), datasource, timeInterval); - List<Map> udpTotalSessionResult = getData(queryService.executeQuery(queryBuild(udpTotalSessionSql))); - - String udpTotalClientIpsSql = String.format(Objects.requireNonNull(env.getProperty("TOTAL_ENTITY_UDP_UNIQ_CLIENT_IPS")), datasource, timeInterval); - List<Map> udpTotalClientIpsResult = getData(queryService.executeQuery(queryBuild(udpTotalClientIpsSql))); - - long tcpSessions = 0; - long tcpClientIps = 0; - long udpSessions = 0; - long udpClientIps = 0; - long tcpTotalSessions = StringUtil.isEmpty(tcpTotalSessionResult.get(0).get("sessions")) ? 0 : Long.parseLong(tcpTotalSessionResult.get(0).get("sessions").toString()); - long tcpTotalClientIps = StringUtil.isEmpty(tcpTotalClientIpResult.get(0).get("uniq_client_ips")) ? 0 : Long.parseLong(tcpTotalClientIpResult.get(0).get("uniq_client_ips").toString()); - long udpTotalSessions = StringUtil.isEmpty(udpTotalSessionResult.get(0).get("sessions")) ? 0 : Long.parseLong(udpTotalSessionResult.get(0).get("sessions").toString()); - long udpTotalClientIps = StringUtil.isEmpty(udpTotalClientIpsResult.get(0).get("uniq_client_ips")) ? 0 : Long.parseLong(udpTotalClientIpsResult.get(0).get("uniq_client_ips").toString()); - for (Map item : tcpSessionResult) { - tcpSessions += StringUtil.isEmpty(item.get("sessions")) ? 0 : Long.parseLong(item.get("sessions").toString()); - } - for (Map item : tcpClientIpsResult) { - tcpClientIps += StringUtil.isEmpty(item.get("client_ips")) ? 0 : Long.parseLong(item.get("client_ips").toString()); - } - for (Map item : udpSessionResult) { - udpSessions += StringUtil.isEmpty(item.get("sessions")) ? 0 : Long.parseLong(item.get("sessions").toString()); - } - for (Map item : udpClientIpsResult) { - udpClientIps += StringUtil.isEmpty(item.get("client_ips")) ? 0 : Long.parseLong(item.get("client_ips").toString()); - } - - String message = StrUtil.format("Top {} Server IPs, {} of TCP sessions; " + - "Top {} Server IPs, {} of TCP uniq client ips; " + - "Top {} Server IPs, {} of UDP(53,443) sessions; " + - "Top {} Server IPs, {} of UDP uniq client ips", - entityConfigSource.getTopServerIPByTCPSession(), tcpTotalSessions == 0 ? "-" : NumberUtil.decimalFormat("#.##%", (tcpSessions * 1.0) / tcpTotalSessions), - entityConfigSource.getTopServerIPByTCPUniqClientIP(), tcpTotalClientIps == 0 ? "-" : NumberUtil.decimalFormat("#.##%", (tcpClientIps * 1.0) / tcpTotalClientIps), - entityConfigSource.getTopServerIPByUDPBySession(), udpTotalSessions == 0 ? "-" : NumberUtil.decimalFormat("#.##%", (udpSessions * 1.0) / udpTotalSessions), - entityConfigSource.getTopServerIPByUDPUniqClientIP(), udpTotalClientIps == 0 ? "-" : NumberUtil.decimalFormat("#.##%", (udpClientIps * 1.0) / udpTotalClientIps)); - return message; - } - - private String getTopSniMessage(String timeInterval, String datasource) { - String topSniSql = String.format(Objects.requireNonNull(env.getProperty("ENTITY_TOP_SNI")), datasource, timeInterval, - entityConfigSource.getTopSNIDefaultSize()); - List<Map> topSniResult = getData(queryService.executeQuery(queryBuild(topSniSql))); - - String totalSniSql = String.format(Objects.requireNonNull(env.getProperty("ENTITY_SNI_TOTAL")), datasource, timeInterval); - List<Map> totalSniResult = getData(queryService.executeQuery(queryBuild(totalSniSql))); - - int cdnCount = 0; - long topSniBytes = 0; - long topSniSessions = 0; - long totalSniBytes = StringUtil.isEmpty(totalSniResult.get(0).get("bytes")) ? 0 : Long.parseLong(totalSniResult.get(0).get("bytes").toString()); - long totalSniSessions = StringUtil.isEmpty(totalSniResult.get(0).get("sessions")) ? 0 : Long.parseLong(totalSniResult.get(0).get("sessions").toString()); - Map<String, List<String>> cdnMap = entityService.buildDomainHashTable(getCfgCDN()); - for (Map item : topSniResult) { - if (entityService.inCDN(cdnMap, String.valueOf(item.get("ssl_sni")))) { - cdnCount++; - } - topSniBytes += StringUtil.isEmpty(item.get("bytes")) ? 0 : Long.parseLong(item.get("bytes").toString()); - topSniSessions += StringUtil.isEmpty(item.get("sessions")) ? 0 : Long.parseLong(item.get("sessions").toString()); - } - - String message = StrUtil.format("Top {} SNI, {} of ssl bytes, {} of ssl sessions, {} popular CDNs", - entityConfigSource.getTopSNIDefaultSize(), - totalSniBytes == 0 ? "-" : NumberUtil.decimalFormat("#.##%", (topSniBytes * 1.0) / totalSniBytes), - totalSniSessions == 0 ? "-" : NumberUtil.decimalFormat("#.##%", (topSniSessions * 1.0) / totalSniSessions), - cdnCount == 0 ? "-" : cdnCount); - - return message; - } - - - private String getTimeInterval(int defaultInterval) { - Date now = new Date(); - Date someMinute = DateUtils.getSomeMinute(now, defaultInterval); - String end = DateUtils.getFormatDate(now, DateUtils.YYYY_MM_DD_HH24_MM_SS); - String start = DateUtils.getFormatDate(someMinute, DateUtils.YYYY_MM_DD_HH24_MM_SS); - List<String> list = Lists.newArrayList(start.concat("/").concat(end)); - StringBuilder timeInterval = new StringBuilder(); - String[] intervals = getIntervals(list); - timeInterval.append("recv_time >= UNIX_TIMESTAMP('").append(intervals[0]).append("') AND recv_time< UNIX_TIMESTAMP('").append(intervals[1]).append("')"); - return timeInterval.toString(); - } - - private String[] getIntervals(List<String> intervals) { - return intervals.get(0).split("/"); - } public List<Map> getData(BaseResult baseResult) { List<Map> dataList = (List<Map>) baseResult.getData(); return dataList; } - private Map<String, List<String>> getCfgCDN() { - Object codeInfo = metadataService.getCfg("public_code_info.json"); - if (StringUtil.isNotEmpty(codeInfo)) { - Map<String, Map<String, List<String>>> data = JSON.parseObject(codeInfo.toString(), Map.class); - return Maps.newHashMap(data.get("CDN")); - } - return Maps.newHashMap(); - } - - private Map<String, Object> generateDatasets(String category) { - List<String> categorylist = Lists.newLinkedList(); - List<Record> list = Db.find(String.format(Objects.requireNonNull(env.getProperty("SQL_DATASETS_CATEGORY")))); - list.forEach(record -> categorylist.add(String.valueOf(record.getColumns().get("category")))); - if (CollectionUtil.isNotEmpty(categorylist) && StrUtil.isNotBlank(category) && !categorylist.contains(category)) { - throw new QGWBusinessException(HttpStatus.HTTP_BAD_REQUEST, ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.DIAGNOSIS_CATEGORY_ERROR)); - } - return sqlDatasetService.getDatasets(Lists.newArrayList(), category, null); - } - - private Map<String, Map<String,String>> buildResult(String executeEngine,List<String> list,boolean isSaved, String option) { - Map<String, Map<String, String>> resultMap = Maps.newLinkedHashMap(); - Map<String, String> infoMap = Maps.newLinkedHashMap(); - FileWriter writer = null; - String benchFileName = executeEngine + "_queries_" + DateUtils.getCurrentDate(DateUtils.YYYYMMDD) + ".sql"; - if (isSaved && DiagnosisOptionEnum.EXECUTION.getValue().equalsIgnoreCase(option)) { - File file = new File("benchmark" + File.separator + benchFileName + "." + IdUtil.simpleUUID() + ".dat"); - writer = FileWriter.create(file); - } - int index = 0; - for (String line : list) { - index++; - Stopwatch watch = Stopwatch.createStarted(); - BaseResult queryResult = sqlDatasetService.getPreview(line, option); - if (StringUtil.isEmpty(queryResult.getStatistics())) { - Map<String, Object> statistics = Maps.newLinkedHashMap(); - statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); - queryResult.setStatistics(statistics); - } - processQueryResult(benchFileName, infoMap, writer, index, queryResult); - } - resultMap.put(benchFileName, buildQueryTimeMetric(infoMap)); - return resultMap; - } - - private Map<String, String> buildQueryTimeMetric(Map<String, String> infoMap) { - if (StringUtil.isNotEmpty(infoMap)) { - double[] values = infoMap.values().stream().mapToDouble(value -> Double.valueOf(value)).sorted().toArray(); - infoMap.put("Min", String.valueOf(CommonUtil.round(StatUtils.min(values),2))); - infoMap.put("Mean", String.valueOf(CommonUtil.round(StatUtils.mean(values),2))); - infoMap.put("Median", String.valueOf(CommonUtil.round(StatUtils.percentile(values, 50),2))); - infoMap.put("P95", String.valueOf(CommonUtil.round(StatUtils.percentile(values, 95),2))); - infoMap.put("P99", String.valueOf(CommonUtil.round(StatUtils.percentile(values, 99),2))); - infoMap.put("MAX", String.valueOf(CommonUtil.round(StatUtils.max(values),2))); - } - return infoMap; - } - - private void processQueryResult(String name, Map<String, String> infoMap, FileWriter writer, int index, BaseResult queryResult) { - int statusCode = Integer.parseInt(queryResult.getStatus().toString()); - if (statusCode == ResultStatusEnum.SUCCESS.getCode()) { - if (writer != null) { - writer.append("--Query" + index + "\t" + "Elapsed(ms):" + queryResult.getStatistics().get("elapsed") + "\n"); - writer.append(queryResult.getMeta() + "\n"); - writer.append(queryResult.getData() + "\n"); - } - infoMap.put("Query" + index, queryResult.getStatistics().get("elapsed") + ""); - } else if (statusCode == ResultStatusEnum.GATEWAY_TIMEOUT.getCode() || statusCode == ResultStatusEnum.BAD_GATEWAY.getCode()) { - infoMap.put("Query" + index, "status:" + queryResult.getStatus() + ",message:" + queryResult.getMessage()); - - } else { - throw new QGWBusinessException(statusCode, QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getMessage(),name - + ",Query" + index + "," + queryResult.getStatus() + "," + queryResult.getMessage())); - } - } - - private QueryProfile queryBuild(String sql) { - return QueryProfile.builder().format(QueryFormatEnum.JSON.getValue()).query(sql).build(); - } private Map<String, String> getPocSqlByDatasource(Optional<File> file, Integer queryNo) { Map<String, String> infoMap = Maps.newLinkedHashMap(); @@ -694,91 +168,6 @@ DiagnosisServiceImpl implements DiagnosisService, EnvironmentAware { /** - * 对指定数据源dbName下的schema 与数据库的字段进行对比检查 - * - * @param dbName - * @param tables - * @return - */ - private Map<String, Map<String, String>> checkMetadataByDatasource(String dbName, List<String> tables) { - - Map<String, Map<String, String>> resultMap = Maps.newLinkedHashMap(); - Map<String, String> infoMap = Maps.newHashMap(); - for (String tableName : tables) { - Schema schema = metadataService.getSchemaByName(tableName); - if (!dbName.equals(schema.getNamespace())) continue; - BaseResult baseResult = getBaseResultByEngine(QueryProfile.builder().option(QueryOptionEnum.REAL_TIME.getValue()).query("describe " + tableName).build()); - if (ResultStatusEnum.SUCCESS.getCode() != baseResult.getStatus()) { - log.error("The Table[" + tableName + "] Structure query failed" + baseResult.getMessage()); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getMessage()," The Table[" + tableName + "] Structure query failed")); - } - List<Map<String, Object>> dbFields = JsonPath.read(JSON.toJSONString(baseResult.getData()), "$.fields.*"); - - Map<String, Object> dbFieldMap = Maps.newHashMap(); - dbFields.stream().forEach(objectMap -> { - dbFieldMap.put(objectMap.get("name").toString(), objectMap.get("type")); - }); - List<Map<String, Object>> schemaFields = JsonPath.read(schema.toString(), "$.fields.*"); - Map<String, Object> schemaFieldMap = Maps.newHashMap(); - schemaFields.stream().forEach(objectMap -> { - schemaFieldMap.put(objectMap.get("name").toString(), objectMap.get("type")); - }); - - MapDifference<String, Object> difference = Maps.difference(schemaFieldMap, dbFieldMap); - - Map<String, Object> entriesOnlyOnLeft = difference.entriesOnlyOnLeft(); - - if (entriesOnlyOnLeft.size() > 0) { - log.error("Schema Fields greater than Table Fields,Schema " + tableName + " exist fields :" + entriesOnlyOnLeft); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - QGWErrorCode.SQL_QUERY_FEDERATION_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_QUERY_FEDERATION_EXCEPTION.getMessage(), - " Schema Fields greater than Table Fields,Schema " + tableName + " exist fields " + entriesOnlyOnLeft)); - - } - Map<String, Object> entriesOnlyOnRight = difference.entriesOnlyOnRight(); - if (entriesOnlyOnRight.size() > 0) { - log.warn("键只存在于右边Map的映射项:" + entriesOnlyOnRight); - infoMap.put(tableName, entriesOnlyOnRight.toString()); - } - - } - resultMap.put(METADATA_PREFIX + dbName, infoMap); - return resultMap; - } - - private void checkCKIndexKey(String dbName, List<String> tables) { - for (String tableName : tables) { - Schema schema = metadataService.getSchemaByName(tableName); - if (!dbName.equals(schema.getNamespace())) continue; - BaseResult indexKeyResult = getBaseResultByEngine(QueryProfile.builder().option(QueryOptionEnum.REAL_TIME.getValue()).query(String.format(Objects.requireNonNull(env.getProperty("TABLE_INDEX_KEY")), tableName, clickHouseHttpSource.getDbName())).build()); - if (ResultStatusEnum.SUCCESS.getCode() != indexKeyResult.getStatus()) { - log.error("The Table[" + tableName + "] index key query failed" + indexKeyResult.getMessage()); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_EXECUTION_SERVER_EXCEPTION.getMessage(), " The Table[" + tableName + "] index key query failed")); - } - List<Map<String, Object>> dataList = (List<Map<String, Object>>) indexKeyResult.getData(); - List<String> dbIndexKey = StringUtil.isEmpty(dataList.get(0)) ? Lists.newArrayList() : (List<String>) dataList.get(0).get("index_key"); - List<String> schemaIndexKey = metadataService.getIndexKey(tableName); - if (schemaIndexKey.equals(dbIndexKey)) { - continue; - } - log.error("{} schema index key inconsistent with DB, schema: {}, db: {}", tableName, schemaIndexKey, dbIndexKey); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - QGWErrorCode.SCHEMA_WITCH_DB_INCONSISTENTY.getCode(), - String.format(QGWErrorCode.SCHEMA_WITCH_DB_INCONSISTENTY.getMessage(), - String.format("%s schema index key is: %s, db index key is: %s", tableName, schemaIndexKey, dbIndexKey))); - } - } - private BaseResult getBaseResultByEngine(QueryProfile queryProfile) { - return queryService.executeQuery(queryProfile); - } - - - /** * 对指定的数据源dbName下的schema进行静态检查 * * @param dbName @@ -789,9 +178,8 @@ DiagnosisServiceImpl implements DiagnosisService, EnvironmentAware { Map<String, Map<String, String>> resultMap = Maps.newLinkedHashMap(); Map<String, String> infoMap = Maps.newHashMap(); for (String tableName : tables) { - Schema schema = metadataService.getSchemaByName(tableName); - Map schemaMap = JSON.parseObject(schema.toString(), Map.class); - if (!dbName.equals(schema.getNamespace())) continue; + LinkedHashMap<String, Object> schemaMap = databaseService.getSchemaByName(tableName); + if (!dbName.equals(schemaMap.get("namespace"))) continue; List<Map<String, Object>> fields = (List<Map<String, Object>>) schemaMap.get("fields"); infoMap.put(tableName, fields.size() + " Fields"); } diff --git a/src/main/java/com/mesalab/qgw/service/impl/DslServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/DslServiceImpl.java index 294b4029..0f61db36 100644 --- a/src/main/java/com/mesalab/qgw/service/impl/DslServiceImpl.java +++ b/src/main/java/com/mesalab/qgw/service/impl/DslServiceImpl.java @@ -11,32 +11,43 @@ import com.geedgenetworks.utils.StringUtil; import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.mesalab.cn.enums.RangeTypeEnum; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.OutputMode; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.enums.QueryOption; import com.mesalab.common.exception.BusinessException; import com.mesalab.common.utils.TreeUtils; -import com.mesalab.network.common.Constants; -import com.mesalab.network.model.protocol.ProtocolTree; -import com.mesalab.qgw.model.basic.DSLProfile; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.service.DslService; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.qgw.service.QueryService; +import com.mesalab.knowledge.entity.*; +import com.mesalab.knowledge.enums.MatchEnum; +import com.mesalab.knowledge.enums.SortEnum; +import com.mesalab.knowledge.service.KnowledgeService; +import com.mesalab.qgw.constant.dsl.LiveChartConstants; +import com.mesalab.qgw.model.dsl.LiveChartProtocol; +import com.mesalab.qgw.constant.DslIdentifierNameConst; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.DSLQueryContext; +import com.mesalab.qgw.model.basic.SQLQueryContext; +import com.mesalab.qgw.service.DSLService; +import com.mesalab.qgw.service.DatabaseService; +import com.mesalab.qgw.service.SQLSyncQueryService; import com.mesalab.services.common.property.SqlPropertySourceFactory; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.LongValue; import net.sf.jsqlparser.expression.StringValue; -import net.sf.jsqlparser.expression.operators.relational.ExpressionList; -import net.sf.jsqlparser.expression.operators.relational.InExpression; -import net.sf.jsqlparser.expression.operators.relational.ItemsList; +import net.sf.jsqlparser.expression.operators.relational.*; import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.PropertySource; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import org.springframework.web.bind.annotation.RequestBody; import java.math.BigDecimal; import java.util.*; @@ -51,61 +62,80 @@ import java.util.stream.Collectors; */ @Service @PropertySource(value = "classpath:dsl-sql-template.sql", factory = SqlPropertySourceFactory.class) -public class DslServiceImpl implements DslService { +public class DslServiceImpl implements DSLService { private static final Log log = LogFactory.get(); @Autowired Environment environment; @Autowired - MetadataService metadataService; + DatabaseService databaseService; @Autowired - QueryService queryService; + SQLSyncQueryService sqlSyncQueryService; + @Autowired + KnowledgeService knowledgeService; private static final String TABLE_TRAFFIC_GENERAL_STAT = "traffic_general_stat"; private static final String TABLE_APPLICATION_PROTOCOL_STAT = "application_protocol_stat"; private static final String TABLE_SESSION_RECORD = "session_record"; - private static final String TABLE_RELATION_ACCOUNT_FRAMEDIP = "relation_account_framedip"; - private static final String TABLE_GTPC_KNOWLEDGE_BASE = "gtpc_knowledge_base"; + private static final String LOGICAL_TYPE_TIMESTAMP = "timestamp"; private static final String LOGICAL_TYPE_UNIX_TIMESTAMP = "unix_timestamp"; + @Override + public BaseResult execDsl(@RequestBody DSLQueryContext dslProfile, boolean isDryRun) { + log.info("HTTP REST DSL, params is: {}", dslProfile); + if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_SUMMARY.equals(dslProfile.getName())) { + return appAndProtocolSummary(dslProfile, isDryRun); + } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_TREE_COMPOSITION.equals(dslProfile.getName())) { + return appAndProtocolTreeComposition(dslProfile, isDryRun); + } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_TREE_THROUGHPUT.equals(dslProfile.getName())) { + return applicationAndProtocolTreeThroughput(dslProfile, isDryRun); + } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_TOP_APPS.equals(dslProfile.getName())) { + return applicationAndProtocolTopApp(dslProfile, isDryRun); + } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_APP_RELATED_INTERNAL_IPS.equals(dslProfile.getName())) { + return applicationAndProtocolAppRelatedInternalIps(dslProfile, isDryRun); + } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_APP_THROUGHPUT.equals(dslProfile.getName())) { + return applicationAndProtocolAppThroughput(dslProfile, isDryRun); + } else if (DslIdentifierNameConst.APPLICATION_AND_PROTOCOL_APP_SUMMARY.equals(dslProfile.getName())) { + return applicationAndProtocolAppSummary(dslProfile, isDryRun); + } else if (DslIdentifierNameConst.IP_LEARNING_FQDN_RELATE_IP.equals(dslProfile.getName())) { + return ipLearningFqdnRelateIp(dslProfile, isDryRun); + } else if (DslIdentifierNameConst.IP_LEARNING_ACTIVE_IP.equals(dslProfile.getName())) { + return ipLearningActiveIp(dslProfile, isDryRun); + } + throw new BusinessException(HttpStatusCodeEnum.NOT_FOUND.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), dslProfile.getName())); + } @Override - public BaseResult appAndProtocolSummary(DSLProfile dslProfile) { - BaseResult asymmetricFlowsResult = queryService.executeQuery( - QueryProfile.builder().query(dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_SUMMARY_ASYMMETRIC_FLOWS_STAT"), TABLE_TRAFFIC_GENERAL_STAT, metadataService.getPartitionKey(TABLE_TRAFFIC_GENERAL_STAT), LOGICAL_TYPE_TIMESTAMP)).build()); + public BaseResult appAndProtocolSummary(DSLQueryContext dslProfile, boolean isDryRun) { + String queryOption = isDryRun ? QueryOption.SYNTAX_VALIDATION.getValue() : QueryOption.REAL_TIME.getValue(); + BaseResult asymmetricFlowsResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder().originalSQL(dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_SUMMARY_ASYMMETRIC_FLOWS_STAT"), TABLE_TRAFFIC_GENERAL_STAT, databaseService.getPartitionKey(TABLE_TRAFFIC_GENERAL_STAT), LOGICAL_TYPE_TIMESTAMP)).option(queryOption).build()); if (!asymmetricFlowsResult.isSuccess()) { - throw new BusinessException(asymmetricFlowsResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), asymmetricFlowsResult.getMessage())); + throw new BusinessException(asymmetricFlowsResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), asymmetricFlowsResult.getMessage())); } - BaseResult networkTrafficStatResult = queryService.executeQuery( - QueryProfile.builder().query(dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_SUMMARY_TRAFFIC_STAT"), TABLE_APPLICATION_PROTOCOL_STAT, metadataService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP)).build()); + BaseResult networkTrafficStatResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder().originalSQL(dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_SUMMARY_TRAFFIC_STAT"), TABLE_APPLICATION_PROTOCOL_STAT, databaseService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP)).option(queryOption).build()); if (!networkTrafficStatResult.isSuccess()) { - throw new BusinessException(networkTrafficStatResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), networkTrafficStatResult.getMessage())); + throw new BusinessException(networkTrafficStatResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), networkTrafficStatResult.getMessage())); } - BaseResult networkUniqueClientIpStatResult = queryService.executeQuery( - QueryProfile.builder().query(dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_SUMMARY_UNIQUE_CLIENT_IP_STAT"), TABLE_APPLICATION_PROTOCOL_STAT, metadataService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP)).build()); - if (!networkUniqueClientIpStatResult.isSuccess()) { - throw new BusinessException(networkUniqueClientIpStatResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), networkUniqueClientIpStatResult.getMessage())); - } - - BaseResult networkTcpStatResult = queryService.executeQuery( - QueryProfile.builder().query(dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_SUMMARY_TCP_STAT"), TABLE_APPLICATION_PROTOCOL_STAT, metadataService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP)).build()); + BaseResult networkTcpStatResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder().originalSQL(dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_SUMMARY_TCP_STAT"), TABLE_APPLICATION_PROTOCOL_STAT, databaseService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP)).option(queryOption).build()); if (!networkTcpStatResult.isSuccess()) { - throw new BusinessException(networkTcpStatResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), networkTcpStatResult.getMessage())); + throw new BusinessException(networkTcpStatResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), networkTcpStatResult.getMessage())); } - BaseResult networkAppStatResult = queryService.executeQuery( - QueryProfile.builder().query(dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_SUMMARY_APP_STAT"), TABLE_APPLICATION_PROTOCOL_STAT, metadataService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP)).build()); + BaseResult networkAppStatResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder().originalSQL(dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_SUMMARY_APP_STAT"), TABLE_APPLICATION_PROTOCOL_STAT, databaseService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP)).option(queryOption).build()); if (!networkAppStatResult.isSuccess()) { - throw new BusinessException(networkAppStatResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), networkAppStatResult.getMessage())); + throw new BusinessException(networkAppStatResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), networkAppStatResult.getMessage())); } List<Map> asymmetricFlowsData = (List<Map>) asymmetricFlowsResult.getData(); List<Map> trafficStatData = (List<Map>) networkTrafficStatResult.getData(); - List<Map> uniqueClientIpStatData = (List<Map>) networkUniqueClientIpStatResult.getData(); List<Map> tcpData = (List<Map>) networkTcpStatResult.getData(); List<Map> appData = (List<Map>) networkAppStatResult.getData(); @@ -117,9 +147,6 @@ public class DslServiceImpl implements DslService { if (!trafficStatData.isEmpty()) { data.putAll(trafficStatData.get(0)); } - if (!uniqueClientIpStatData.isEmpty()) { - data.putAll(uniqueClientIpStatData.get(0)); - } if (!tcpData.isEmpty()) { data.putAll(tcpData.get(0)); } @@ -128,94 +155,89 @@ public class DslServiceImpl implements DslService { } long asymmetricFlows = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_BYTES) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_BYTES))); + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_BYTES) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_BYTES))); long totalSessionUseOnAsymmetricFlows = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES_USE_ON_ASYMMETRIC_FLOWS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES_USE_ON_ASYMMETRIC_FLOWS))); + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES_USE_ON_ASYMMETRIC_FLOWS) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES_USE_ON_ASYMMETRIC_FLOWS))); - long uniqueClientIp = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_UNIQ_CLIENT_IP) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_UNIQ_CLIENT_IP))); long totalSessions = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS))); + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS))); long totalBytes = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES))); + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES))); long totalPackets = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS))); + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS))); long fragmentationPackets = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS))); + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS))); long dataRate = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_DATA_RATE) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_DATA_RATE))); + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_DATA_RATE) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_DATA_RATE))); long tcpRetransmissionsPackets = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PACKETS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PACKETS))); + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PACKETS) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PACKETS))); long tcpTotalPackets = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_TCP_TOTAL_PACKETS) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_TCP_TOTAL_PACKETS))); + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TCP_TOTAL_PACKETS) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TCP_TOTAL_PACKETS))); long unknownAppBytes = Long.parseLong(String.valueOf( - data.get(Constants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_BYTES) == null ? 0 : data.get(Constants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_BYTES))); - + data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_BYTES) == null ? 0 : data.get(LiveChartConstants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_BYTES))); - String unknownAppPercent = totalBytes == 0 ? String.valueOf(0) - : NumberUtil.div(BigDecimal.valueOf(unknownAppBytes), BigDecimal.valueOf(totalBytes), 4).toPlainString(); - String asymmetricFlowsPercent = totalSessionUseOnAsymmetricFlows == 0 ? String.valueOf(0) - : NumberUtil.div(BigDecimal.valueOf(asymmetricFlows), BigDecimal.valueOf(totalSessionUseOnAsymmetricFlows), 4).toPlainString(); - String tcpRetransmissionsPercent = tcpTotalPackets == 0 ? String.valueOf(0) - : NumberUtil.div(BigDecimal.valueOf(tcpRetransmissionsPackets), BigDecimal.valueOf(tcpTotalPackets), 4).toPlainString(); - String fragmentationPacketsPercent = totalPackets == 0 ? String.valueOf(0) - : NumberUtil.div(BigDecimal.valueOf(fragmentationPackets), BigDecimal.valueOf(totalPackets), 4).toPlainString(); + double unknownAppPercent = totalBytes == 0 ? 0.0 + : NumberUtil.div(BigDecimal.valueOf(unknownAppBytes), BigDecimal.valueOf(totalBytes), 4).doubleValue(); + double asymmetricFlowsPercent = totalSessionUseOnAsymmetricFlows == 0 ? 0.0 + : NumberUtil.div(BigDecimal.valueOf(asymmetricFlows), BigDecimal.valueOf(totalSessionUseOnAsymmetricFlows), 4).doubleValue(); + double tcpRetransmissionsPercent = tcpTotalPackets == 0 ? 0.0 + : NumberUtil.div(BigDecimal.valueOf(tcpRetransmissionsPackets), BigDecimal.valueOf(tcpTotalPackets), 4).doubleValue(); + double fragmentationPacketsPercent = totalPackets == 0 ? 0.0 + : NumberUtil.div(BigDecimal.valueOf(fragmentationPackets), BigDecimal.valueOf(totalPackets), 4).doubleValue(); List result = new ArrayList<>(); Map resultMap = new LinkedHashMap<>(); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_UNIQ_CLIENT_IP, uniqueClientIp); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS, totalSessions); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_DATA_RATE, dataRate); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES, totalBytes); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS, totalPackets); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_BYTES, unknownAppBytes); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_PERCENT, unknownAppPercent); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_BYTES, asymmetricFlows); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_PERCENT, asymmetricFlowsPercent); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PACKETS, tcpRetransmissionsPackets); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PERCENT, tcpRetransmissionsPercent); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS, fragmentationPackets); - resultMap.put(Constants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PERCENT, fragmentationPacketsPercent); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_SESSIONS, totalSessions); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_DATA_RATE, dataRate); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_BYTES, totalBytes); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TOTAL_PACKETS, totalPackets); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_BYTES, unknownAppBytes); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_UNKNOWN_APP_PERCENT, unknownAppPercent); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_BYTES, asymmetricFlows); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_ASYMMETRIC_FLOWS_PERCENT, asymmetricFlowsPercent); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PACKETS, tcpRetransmissionsPackets); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_TCP_RETRANSMISSIONS_PERCENT, tcpRetransmissionsPercent); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PACKETS, fragmentationPackets); + resultMap.put(LiveChartConstants.NETWORK_OVERVIEW_METRIC_FRAGMENTATION_PERCENT, fragmentationPacketsPercent); result.add(resultMap); Map statistics = asymmetricFlowsResult.getStatistics(); Map networkTrafficStatistics = networkTrafficStatResult.getStatistics(); - Map networkUniqueClientIpStatistics = networkUniqueClientIpStatResult.getStatistics(); Map networkTcpStatistics = networkTcpStatResult.getStatistics(); Map networkAppStatistics = networkAppStatResult.getStatistics(); for (Object key : statistics.keySet()) { if ("result_rows".equals(key.toString())) { continue; } - statistics.put(key, Long.parseLong(networkTrafficStatistics.get(key).toString()) + Long.parseLong(networkUniqueClientIpStatistics.get(key).toString()) + Long.parseLong(networkTcpStatistics.get(key).toString()) + Long.parseLong(networkAppStatistics.get(key).toString())); + statistics.put(key, Long.parseLong(networkTrafficStatistics.get(key).toString()) + Long.parseLong(networkTcpStatistics.get(key).toString()) + Long.parseLong(networkAppStatistics.get(key).toString())); } return BaseResultGenerator.success("ok", result, statistics); } @Override - public BaseResult appAndProtocolTreeComposition(DSLProfile dslProfile) { - String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_TREE_COMPOSITION"), TABLE_APPLICATION_PROTOCOL_STAT, metadataService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); + public BaseResult appAndProtocolTreeComposition(DSLQueryContext dslProfile, boolean isDryRun) { + String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_TREE_COMPOSITION"), TABLE_APPLICATION_PROTOCOL_STAT, databaseService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); + BaseResult baseResult = sqlSyncQueryService.executeQuery(queryBuild(sql, isDryRun)); if (!baseResult.isSuccess()) { - throw new BusinessException(baseResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + throw new BusinessException(baseResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); } List<Map> resultList = (List<Map>) baseResult.getData(); if (StringUtil.isEmpty(resultList)) { return BaseResultGenerator.success("ok", resultList, baseResult.getStatistics()); } - List<ProtocolTree> protocolTreeList = buildHierarchicalStructure(buildFlatStructure(resultList)); - return BaseResultGenerator.success("ok", protocolTreeList, baseResult.getStatistics()); + List<LiveChartProtocol> protocols = buildHierarchicalStructure(buildFlatStructure(resultList)); + return BaseResultGenerator.success("ok", protocols, baseResult.getStatistics()); } @Override - public BaseResult applicationAndProtocolTreeThroughput(DSLProfile dslProfile) { - String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_TREE_THROUGHPUT"), TABLE_APPLICATION_PROTOCOL_STAT, metadataService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); + public BaseResult applicationAndProtocolTreeThroughput(DSLQueryContext dslProfile, boolean isDryRun) { + String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_TREE_THROUGHPUT"), TABLE_APPLICATION_PROTOCOL_STAT, databaseService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); + BaseResult baseResult = sqlSyncQueryService.executeQuery(queryBuild(sql, isDryRun)); if (!baseResult.isSuccess()) { - throw new BusinessException(baseResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + throw new BusinessException(baseResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); } List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); data.forEach(o -> { @@ -228,87 +250,361 @@ public class DslServiceImpl implements DslService { } @Override - public BaseResult applicationAndProtocolTopApp(DSLProfile dslProfile) { + public BaseResult applicationAndProtocolTopApp(DSLQueryContext dslProfile, boolean isDryRun) { if (StrUtil.isEmpty(dslProfile.getGranularity())) { dslProfile.setGranularity("PT15S"); } - String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_TOP_APPS"), TABLE_APPLICATION_PROTOCOL_STAT, metadataService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); + String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_TOP_APPS"), TABLE_APPLICATION_PROTOCOL_STAT, databaseService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); + BaseResult baseResult = sqlSyncQueryService.executeQuery(queryBuild(sql, isDryRun)); if (!baseResult.isSuccess()) { - throw new BusinessException(baseResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + throw new BusinessException(baseResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); } return baseResult; } @Override - public BaseResult applicationAndProtocolAppRelatedInternalIps(DSLProfile dslProfile) { - String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_APP_RELATED_INTERNAL_IPS"), TABLE_SESSION_RECORD, metadataService.getPartitionKey(TABLE_SESSION_RECORD), LOGICAL_TYPE_UNIX_TIMESTAMP); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); + public BaseResult applicationAndProtocolAppRelatedInternalIps(DSLQueryContext dslProfile, boolean isDryRun) { + String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_APP_RELATED_INTERNAL_IPS"), TABLE_SESSION_RECORD, databaseService.getPartitionKey(TABLE_SESSION_RECORD), LOGICAL_TYPE_UNIX_TIMESTAMP); + BaseResult baseResult = sqlSyncQueryService.executeQuery(queryBuild(sql, isDryRun)); if (!baseResult.isSuccess()) { - throw new BusinessException(baseResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + throw new BusinessException(baseResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); } return baseResult; } @Override - public BaseResult applicationAndProtocolAppThroughput(DSLProfile dslProfile) { - String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_APP_THROUGHPUT"), TABLE_APPLICATION_PROTOCOL_STAT, metadataService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); + public BaseResult applicationAndProtocolAppThroughput(DSLQueryContext dslProfile, boolean isDryRun) { + String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_APP_THROUGHPUT"), TABLE_APPLICATION_PROTOCOL_STAT, databaseService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); + BaseResult baseResult = sqlSyncQueryService.executeQuery(queryBuild(sql, isDryRun)); if (!baseResult.isSuccess()) { - throw new BusinessException(baseResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + throw new BusinessException(baseResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); } return baseResult; } @Override - public BaseResult applicationAndProtocolAppSummary(DSLProfile dslProfile) { - String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_APP_SUMMARY"), TABLE_APPLICATION_PROTOCOL_STAT, metadataService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); + public BaseResult applicationAndProtocolAppSummary(DSLQueryContext dslProfile, boolean isDryRun) { + String sql = dslProfile.toSql(environment.getProperty("APPLICATION_AND_PROTOCOL_APP_SUMMARY"), TABLE_APPLICATION_PROTOCOL_STAT, databaseService.getPartitionKey(TABLE_APPLICATION_PROTOCOL_STAT), LOGICAL_TYPE_TIMESTAMP); + BaseResult baseResult = sqlSyncQueryService.executeQuery(queryBuild(sql, isDryRun)); if (!baseResult.isSuccess()) { - throw new BusinessException(baseResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + throw new BusinessException(baseResult.getStatus(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); } return baseResult; } @Override - public BaseResult realTimeDataAnalyticsSubscriberIdRelateIp(DSLProfile dslProfile) { + public BaseResult ipLearningFqdnRelateIp(DSLQueryContext dslProfile, boolean isDryRun) { + String queryType = "iplearning"; + String dataSource = "IP_LEARNING_VIEW"; + DSLObject dslObject = getDslObject(dslProfile, queryType, dataSource); + log.info("DSLProfile parse to DSLObject: {}", JSON.toJSONString(dslObject)); + return knowledgeService.query(dslObject); + } + + @Override + public BaseResult ipLearningActiveIp(DSLQueryContext dslProfile, boolean isDryRun) { + String queryType = "ippool"; + String dataSource = "IP_VIEW"; + DSLObject dslObject = getDslObject(dslProfile, queryType, dataSource); + log.info("DSLProfile parse to DSLObject: {}", JSON.toJSONString(dslObject)); + return knowledgeService.query(dslObject); + } + + private static DSLObject getDslObject(DSLQueryContext dslProfile, String queryType, String dataSource) { + Parameters parameters = new Parameters(); + List<String> intervals = dslProfile.getIntervals(); + if (!CollectionUtils.isEmpty(intervals)) { + parameters.setIntervals(intervals); + } + String limit = dslProfile.getLimit(); + if (StrUtil.isNotEmpty(limit)) { + parameters.setLimit(limit); + } + List<Match> matches = Lists.newArrayList(); + List<Range> ranges = Lists.newArrayList(); + ExpressionVisitorAdapter expressionVisitorAdapter = getExpressionVisitorAdapter(matches, ranges); String filter = dslProfile.getFilter(); - if (StrUtil.isNotEmpty(filter) && filter.contains("subscriber_id")) { - try { - dslProfile.setFilter(inToLike(filter)); - } catch (JSQLParserException e) { - log.error("InExpression to LikeExpression error: {}", e.getMessage()); - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), e.getMessage())); + Expression expression = null; + try { + expression = CCJSqlParserUtil.parseExpression(filter, false); + } catch (JSQLParserException e) { + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), e.getMessage())); + } + expression.accept(expressionVisitorAdapter); + Iterator<Range> iterator = ranges.iterator(); + while (iterator.hasNext()) { + Range next = iterator.next(); + if (next.getFieldKey().equals("1") && next.getType().equals(RangeTypeEnum.EQ.getType())) { + iterator.remove(); } } - String sql = dslProfile.toSql(environment.getProperty("REAL_TIME_DATA_ANALYTICS_SUBSCRIBER_ID_RELATE_IP"), TABLE_RELATION_ACCOUNT_FRAMEDIP, "last_update_time", LOGICAL_TYPE_UNIX_TIMESTAMP); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); - if (!baseResult.isSuccess()) { - throw new BusinessException(baseResult.getStatus(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + parameters.setMatch(matches); + parameters.setRange(ranges); + + List<Sort> sorts = Lists.newArrayList(); + if (StrUtil.isNotEmpty(dslProfile.getOrderBy())) { + buildSorts(dslProfile.getOrderBy(), sorts); } - return baseResult; + parameters.setSort(sorts); + DSLObject dslObject = new DSLObject(); + DSLObject.DSLQuery dslQuery = new DSLObject.DSLQuery(); + dslQuery.setQueryType(queryType); + dslQuery.setDataSource(dataSource); + dslQuery.setParameters(parameters); + dslObject.setQuery(dslQuery); + + return dslObject; } - @Override - public BaseResult realTimeDataAnalyticsMobileIdentityRelateTeid(DSLProfile dslProfile) { - return BaseResultGenerator.failure(); + private static void buildSorts(String orderByStr, List<Sort> sorts) { + try { + Statement parse = CCJSqlParserUtil.parse("select 1 from table order by " + orderByStr); + if (parse instanceof Select && ((Select) parse).getSelectBody() instanceof PlainSelect) { + List<OrderByElement> orderByElements = ((PlainSelect) ((Select) parse).getSelectBody()).getOrderByElements(); + for (int i = 0; i < orderByElements.size(); i++) { + OrderByElement orderByElement = orderByElements.get(i); + Sort sort = new Sort(); + sort.setFieldKey(orderByElement.getExpression().toString()); + sort.setType(orderByElement.isAsc() ? SortEnum.ASC.getType() : SortEnum.DESC.getType()); + sorts.add(sort); + } + } + } catch (JSQLParserException e) { + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), e.getMessage())); + } + } + + private static ExpressionVisitorAdapter getExpressionVisitorAdapter(List<Match> matches, List<Range> ranges) { + ExpressionVisitorAdapter expressionVisitorAdapter = new ExpressionVisitorAdapter() { + @Override + public void visit(LikeExpression expr) { + String left = String.valueOf(expr.getLeftExpression()); + String right = ((StringValue) expr.getRightExpression()).getValue(); + String type = ""; + if (right.startsWith("%") && right.endsWith("%")) { + type = MatchEnum.SUBSTRING.getType(); + right = right.substring(1, right.length() - 1); + } else if (right.startsWith("%")) { + type = MatchEnum.SUFFIX.getType(); + right = right.substring(1); + } else if (right.endsWith("%")) { + type = MatchEnum.PREFIX.getType(); + right = right.substring(0, right.length() - 1); + } else { + type = MatchEnum.EXACTLY.getType(); + } + + String finalType = type; + if (matches.stream().anyMatch(o -> o.getFieldKey().equals(left) && o.getType().equals(finalType))) { + for (Match match : matches) { + if (match.getFieldKey().equals(left) && match.getType().equals(type)) { + match.getFieldValues().add(right); + } + } + } else { + Match match = new Match(); + match.setFieldKey(left); + match.setType(type); + match.setFieldValues(Lists.newArrayList(right)); + matches.add(match); + } + } + + @Override + public void visit(EqualsTo expr) { + String left = String.valueOf(expr.getLeftExpression()); + Object right; + if (expr.getRightExpression() instanceof LongValue) { + right = ((LongValue) expr.getRightExpression()).getValue(); + } else { + right = ((StringValue) expr.getRightExpression()).getValue(); + } + if (ranges.stream().anyMatch(o -> o.getFieldKey().equals(left) && o.getType().equals(RangeTypeEnum.EQ.getType()))) { + for (Range range : ranges) { + if (range.getFieldKey().equals(left) && range.getType().equals(RangeTypeEnum.EQ.getType())) { + range.getFieldValues().add(right); + } + } + } else { + Range range = new Range(); + range.setFieldKey(left); + range.setType(RangeTypeEnum.EQ.getType()); + range.setFieldValues(Lists.newArrayList(right)); + ranges.add(range); + } + } + + @Override + public void visit(InExpression expr) { + ItemsList rightItemsList = expr.getRightItemsList(); + if (rightItemsList instanceof ExpressionList && ((ExpressionList) rightItemsList).getExpressions().get(0) instanceof LongValue) { + intParseIn(expr); + } else { + strParseIn(expr); + } + } + + private void intParseIn(InExpression expr) { + String left = String.valueOf(expr.getLeftExpression()); + if (ranges.stream().anyMatch(o -> o.getFieldKey().equals(left) && o.getType().equals(RangeTypeEnum.EQ.getType()))) { + for (Range range : ranges) { + if (range.getFieldKey().equals(left) && range.getType().equals(RangeTypeEnum.EQ.getType())) { + ItemsList rightItemsList = expr.getRightItemsList(); + rightItemsList.accept(new ItemsListVisitorAdapter() { + @Override + public void visit(ExpressionList expressionList) { + for (Expression expression : expressionList.getExpressions()) { + if (expression instanceof StringValue) { + range.getFieldValues().add(((StringValue) expression).getValue()); + } else if (expression instanceof LongValue) { + range.getFieldValues().add(((LongValue) expression).getValue()); + } + + } + } + + }); + } + } + } else { + Range range = new Range(); + range.setFieldKey(left); + range.setFieldValues(Lists.newArrayList()); + range.setType(RangeTypeEnum.EQ.getType()); + ItemsList rightItemsList = expr.getRightItemsList(); + rightItemsList.accept(new ItemsListVisitorAdapter() { + @Override + public void visit(ExpressionList expressionList) { + for (Expression expression : expressionList.getExpressions()) { + if (expression instanceof StringValue) { + range.getFieldValues().add(((StringValue) expression).getValue()); + } else if (expression instanceof LongValue) { + range.getFieldValues().add(((LongValue) expression).getValue()); + } + + } + } + }); + ranges.add(range); + } + } + + private void strParseIn(InExpression expr) { + String left = String.valueOf(expr.getLeftExpression()); + if (matches.stream().anyMatch(o -> o.getFieldKey().equals(left) && o.getType().equals(MatchEnum.EXACTLY.getType()))) { + for (Match match : matches) { + if (match.getFieldKey().equals(left) && match.getType().equals(MatchEnum.EXACTLY.getType())) { + ItemsList rightItemsList = expr.getRightItemsList(); + rightItemsList.accept(new ItemsListVisitorAdapter() { + @Override + public void visit(ExpressionList expressionList) { + for (Expression expression : expressionList.getExpressions()) { + if (expression instanceof StringValue) { + match.getFieldValues().add(((StringValue) expression).getValue()); + } else if (expression instanceof LongValue) { + match.getFieldValues().add(((LongValue) expression).getValue()); + } + + } + } + + }); + } + } + } else { + Match match = new Match(); + match.setFieldKey(left); + match.setFieldValues(Lists.newArrayList()); + match.setType(MatchEnum.EXACTLY.getType()); + ItemsList rightItemsList = expr.getRightItemsList(); + rightItemsList.accept(new ItemsListVisitorAdapter() { + @Override + public void visit(ExpressionList expressionList) { + for (Expression expression : expressionList.getExpressions()) { + if (expression instanceof StringValue) { + match.getFieldValues().add(((StringValue) expression).getValue()); + } else if (expression instanceof LongValue) { + match.getFieldValues().add(((LongValue) expression).getValue()); + } + + } + } + }); + matches.add(match); + } + } + + @Override + public void visit(GreaterThan expr) { + String left = String.valueOf(expr.getLeftExpression()); + Object right; + if (expr.getRightExpression() instanceof LongValue) { + right = ((LongValue) expr.getRightExpression()).getValue(); + } else { + right = ((StringValue) expr.getRightExpression()).getValue(); + } + if (ranges.stream().anyMatch(o -> o.getFieldKey().equals(left) && o.getType().equals(RangeTypeEnum.GT.getType()))) { + for (Range range : ranges) { + if (range.getFieldKey().equals(left) && range.getType().equals(RangeTypeEnum.EQ.getType())) { + range.getFieldValues().add(right); + } + } + } else { + Range range = new Range(); + range.setFieldKey(left); + range.setType(RangeTypeEnum.GT.getType()); + range.setFieldValues(Lists.newArrayList(right)); + ranges.add(range); + } + } + + @Override + public void visit(GreaterThanEquals expr) { + String left = String.valueOf(expr.getLeftExpression()); + Object right; + if (expr.getRightExpression() instanceof LongValue) { + right = ((LongValue) expr.getRightExpression()).getValue(); + } else { + right = ((StringValue) expr.getRightExpression()).getValue(); + } + if (ranges.stream().anyMatch(o -> o.getFieldKey().equals(left) && o.getType().equals(RangeTypeEnum.EQ.getType()))) { + for (Range range : ranges) { + if (range.getFieldKey().equals(left) && range.getType().equals(RangeTypeEnum.EQ.getType())) { + range.getFieldValues().add(right); + } + } + } else { + Range range = new Range(); + range.setFieldKey(left); + range.setType(RangeTypeEnum.EQ.getType()); + range.setFieldValues(Lists.newArrayList(right)); + ranges.add(range); + } + } + }; + return expressionVisitorAdapter; + } + + private SQLQueryContext queryBuild(String sql, boolean isDryRun) { + String queryOption = isDryRun ? QueryOption.SYNTAX_VALIDATION.getValue() : QueryOption.REAL_TIME.getValue(); + return SQLQueryContext.builder().format(OutputMode.JSON.getValue()).originalSQL(sql).option(queryOption).build(); } @Override - public List<ProtocolTree> buildFlatStructure(List<Map> protocolData) { - List<ProtocolTree> nodes = Lists.newArrayList(); - for (Map protocolMap : protocolData) { - ProtocolTree protocolTree = convertStringToObject(JSON.toJSONString(protocolMap, JSONWriter.Feature.LargeObject)); - if (StringUtil.isNotEmpty(protocolTree)) { - nodes.add(protocolTree); + public List<LiveChartProtocol> buildFlatStructure(List<Map> protocols) { + List<LiveChartProtocol> nodes = Lists.newArrayList(); + for (Map protocolMap : protocols) { + LiveChartProtocol protocol = convertStringToObject(JSON.toJSONString(protocolMap, JSONWriter.Feature.LargeObject)); + if (StringUtil.isNotEmpty(protocol)) { + nodes.add(protocol); } } - ProtocolTree root = new ProtocolTree(Constants.PROTOCOL_TREE_ROOT_NAME, Constants.PROTOCOL_TREE_ROOT_NAME, null); - Map<String, Long> protocolsAndApplications = nodes.stream().collect(Collectors.groupingBy(ProtocolTree::getName, Collectors.counting())); - root.addMetric(Constants.PROTOCOL_TREE_METRIC_ENCAPSULATION_PATHS, nodes.size()); - root.addMetric(Constants.PROTOCOL_TREE_METRIC_PROTOCOLS_AND_APPLICATIONS, protocolsAndApplications.size()); - List<ProtocolTree> roots = nodes.stream().filter(o -> StringUtil.isBlank(o.getParentId())).collect(Collectors.toList()); + LiveChartProtocol root = new LiveChartProtocol(LiveChartConstants.PROTOCOL_TREE_ROOT_NAME, LiveChartConstants.PROTOCOL_TREE_ROOT_NAME, null); + Map<String, Long> protocolsAndApplications = nodes.stream().collect(Collectors.groupingBy(LiveChartProtocol::getName, Collectors.counting())); + root.addMetric(LiveChartConstants.PROTOCOL_TREE_METRIC_ENCAPSULATION_PATHS, nodes.size()); + root.addMetric(LiveChartConstants.PROTOCOL_TREE_METRIC_PROTOCOLS_AND_APPLICATIONS, protocolsAndApplications.size()); + List<LiveChartProtocol> roots = nodes.stream().filter(o -> StringUtil.isBlank(o.getParentId())).collect(Collectors.toList()); roots.forEach(item -> { root.setSentBytes(root.getSentBytes() + item.getSentBytes()); root.setReceivedBytes(root.getReceivedBytes() + item.getReceivedBytes()); @@ -316,16 +612,16 @@ public class DslServiceImpl implements DslService { root.setTotalReceivedBytes(root.getTotalReceivedBytes() + item.getReceivedBytes()); }); nodes.forEach(item -> { - item.setId(Constants.PROTOCOL_TREE_ROOT_NAME + Constants.ENCAPSULATION_PATH_SEPARATOR + item.getId()); + item.setId(LiveChartConstants.PROTOCOL_TREE_ROOT_NAME + LiveChartConstants.ENCAPSULATION_PATH_SEPARATOR + item.getId()); }); nodes.add(root); return nodes; } @Override - public List<ProtocolTree> buildHierarchicalStructure(List<ProtocolTree> nodes) { + public List<LiveChartProtocol> buildHierarchicalStructure(List<LiveChartProtocol> nodes) { - List<ProtocolTree> targetNodes = groupByName(nodes); + List<LiveChartProtocol> targetNodes = groupByName(nodes); targetNodes.sort((o1, o2) -> { long numThis = o1.getReceivedBytes() + o1.getSentBytes(); @@ -338,9 +634,9 @@ public class DslServiceImpl implements DslService { return 0; } }); - Map<String, List<ProtocolTree>> protocolMap = Maps.newLinkedHashMap(); - List<ProtocolTree> roots = targetNodes.stream().filter(o -> StringUtil.isBlank(o.getParentId())).collect(Collectors.toList()); - protocolMap.put(Constants.PROTOCOLS_PARENT_ID, roots); + Map<String, List<LiveChartProtocol>> protocolMap = Maps.newLinkedHashMap(); + List<LiveChartProtocol> roots = targetNodes.stream().filter(o -> StringUtil.isBlank(o.getParentId())).collect(Collectors.toList()); + protocolMap.put(LiveChartConstants.PROTOCOLS_PARENT_ID, roots); targetNodes.stream().filter(o -> StringUtil.isNotBlank(o.getParentId())).forEach(x -> { if (StringUtil.isEmpty(protocolMap.get(x.getParentId()))) { protocolMap.put(x.getParentId(), Lists.newArrayList(x)); @@ -349,18 +645,18 @@ public class DslServiceImpl implements DslService { } }); - List<ProtocolTree> protocolTreeList = TreeUtils.mergeTree(protocolMap, Constants.PROTOCOLS_PARENT_ID, ProtocolTree::getId, ProtocolTree::setChildrens); - for (ProtocolTree protocolTree : protocolTreeList) { - List<ProtocolTree> ethernetList = protocolTree.getChildrens().stream().filter( - p -> Constants.PROTOCOL_ETHERNET_NODE.equals(p.getName())).collect(Collectors.toList()); + List<LiveChartProtocol> protocols = TreeUtils.mergeTree(protocolMap, LiveChartConstants.PROTOCOLS_PARENT_ID, LiveChartProtocol::getId, LiveChartProtocol::setChildrens); + for (LiveChartProtocol protocolTree : protocols) { + List<LiveChartProtocol> ethernetList = protocolTree.getChildrens().stream().filter( + p -> LiveChartConstants.PROTOCOL_ETHERNET_NODE.equals(p.getName())).collect(Collectors.toList()); if (protocolTree.getChildrens().size() != ethernetList.size()) { protocolTree.setChildrens(ethernetList); - protocolTree.setReceivedBytes(StringUtil.isEmpty(ethernetList.get(0)) ? 0 : ethernetList.get(0).getReceivedBytes()); - protocolTree.setSentBytes(StringUtil.isEmpty(ethernetList.get(0)) ? 0 : ethernetList.get(0).getSentBytes()); - log.warn("Protocol Tree Exceptional data occurs and non-{} protocols are filtered out: {}", Constants.PROTOCOL_ETHERNET_NODE, protocolTree.getChildrens()); + protocolTree.setReceivedBytes(ethernetList.isEmpty() || StringUtil.isEmpty(ethernetList.get(0)) ? 0 : ethernetList.get(0).getReceivedBytes()); + protocolTree.setSentBytes(ethernetList.isEmpty() || StringUtil.isEmpty(ethernetList.get(0)) ? 0 : ethernetList.get(0).getSentBytes()); + log.warn("Protocol Tree Exceptional data occurs and non-{} protocols are filtered out: {}", LiveChartConstants.PROTOCOL_ETHERNET_NODE, protocolTree.getChildrens()); } } - return protocolTreeList; + return protocols; } /** @@ -369,23 +665,23 @@ public class DslServiceImpl implements DslService { * @param nodes * @return */ - private List<ProtocolTree> groupByName(List<ProtocolTree> nodes) { - Map<String, List<ProtocolTree>> resultMap = nodes.stream() - .collect(Collectors.groupingBy(ProtocolTree::getName, Collectors.collectingAndThen(Collectors.toList(), protocolTreeList -> { + private List<LiveChartProtocol> groupByName(List<LiveChartProtocol> nodes) { + Map<String, List<LiveChartProtocol>> resultMap = nodes.stream() + .collect(Collectors.groupingBy(LiveChartProtocol::getName, Collectors.collectingAndThen(Collectors.toList(), protocols -> { - List<ProtocolTree> parentNodes = protocolTreeList.stream() + List<LiveChartProtocol> parentNodes = protocols.stream() .filter(p -> StringUtil.isBlank(p.getParentId()) || !Splitter.on(".").omitEmptyStrings().splitToList(p.getParentId()).contains(p.getName())) .collect(Collectors.toList()); Map<String, Long> summedMetrics = Maps.newLinkedHashMap(); Map<String, Long> totalBytesMetrics = Maps.newLinkedHashMap(); - totalBytesMetrics.put(Constants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES, 0L); - totalBytesMetrics.put(Constants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES, 0L); + totalBytesMetrics.put(LiveChartConstants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES, 0L); + totalBytesMetrics.put(LiveChartConstants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES, 0L); - parentNodes.stream().filter(p -> !p.getName().equalsIgnoreCase(Constants.PROTOCOL_TREE_ROOT_NAME)).forEach(protocolTree -> { - totalBytesMetrics.put(Constants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES, totalBytesMetrics.get(Constants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES) + protocolTree.getSentBytes()); - totalBytesMetrics.put(Constants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES, totalBytesMetrics.get(Constants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES) + protocolTree.getReceivedBytes()); + parentNodes.stream().filter(p -> !p.getName().equalsIgnoreCase(LiveChartConstants.PROTOCOL_TREE_ROOT_NAME)).forEach(protocolTree -> { + totalBytesMetrics.put(LiveChartConstants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES, totalBytesMetrics.get(LiveChartConstants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES) + protocolTree.getSentBytes()); + totalBytesMetrics.put(LiveChartConstants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES, totalBytesMetrics.get(LiveChartConstants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES) + protocolTree.getReceivedBytes()); for (String key : protocolTree.getMetrics().keySet()) { - String totalKey = StrUtil.upperFirstAndAddPre(key, "total"); + String totalKey = StrUtil.addPrefixIfNot(key, "total_"); if (StringUtil.isEmpty(summedMetrics.get(totalKey))) { summedMetrics.put(totalKey, Long.valueOf(protocolTree.getMetrics().get(key).toString())); } else { @@ -395,22 +691,23 @@ public class DslServiceImpl implements DslService { } }); - protocolTreeList.stream().filter(p -> !p.getName().equalsIgnoreCase(Constants.PROTOCOL_TREE_ROOT_NAME)).forEach(protocolTree -> { - protocolTree.setTotalSentBytes(totalBytesMetrics.get(Constants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES)); - protocolTree.setTotalReceivedBytes(totalBytesMetrics.get(Constants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES)); + protocols.stream().filter(p -> !p.getName().equalsIgnoreCase(LiveChartConstants.PROTOCOL_TREE_ROOT_NAME)).forEach(protocolTree -> { + protocolTree.setTotalSentBytes(totalBytesMetrics.get(LiveChartConstants.PROTOCOL_TREE_METRIC_TOTAL_SENT_BYTES)); + protocolTree.setTotalReceivedBytes(totalBytesMetrics.get(LiveChartConstants.PROTOCOL_TREE_METRIC_TOTAL_RECEIVED_BYTES)); protocolTree.getMetrics().putAll(summedMetrics); }); - return protocolTreeList; + return protocols; }))); - List<ProtocolTree> protocolTreeList = Lists.newArrayList(); + List<LiveChartProtocol> protocols = Lists.newArrayList(); resultMap.values().forEach(o -> { - protocolTreeList.addAll(o); + protocols.addAll(o); }); - return protocolTreeList; + return protocols; } + private String inToLike(String filter) throws JSQLParserException { List<String> params = Lists.newArrayList(); List<String> oldExpression = Lists.newArrayList(); @@ -433,7 +730,7 @@ public class DslServiceImpl implements DslService { } } }; - Expression expression = CCJSqlParserUtil.parseExpression(filter); + Expression expression = CCJSqlParserUtil.parseExpression(filter, false); String strExpr = expression.toString(); expression.accept(expressionVisitorAdapter); if (params.size() > 0 && oldExpression.size() == 1) { @@ -441,8 +738,9 @@ public class DslServiceImpl implements DslService { } return filter; } - private ProtocolTree convertStringToObject(String protocolString) { - ProtocolTree protocolTree = null; + + private LiveChartProtocol convertStringToObject(String protocolString) { + LiveChartProtocol protocol = null; try { Map<String, Object> results = JSON.parseObject(protocolString, Map.class); String protocolId = String.valueOf(results.get("protocol_stack_id")); @@ -451,28 +749,28 @@ public class DslServiceImpl implements DslService { return null; } long sessions = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_SESSIONS)).orElse(0).toString()); + results.get(LiveChartConstants.PROTOCOL_TREE_METRIC_SESSIONS)).orElse(0).toString()); long sentBytes = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_C2S_BYTES)).orElse(0).toString()); + results.get(LiveChartConstants.PROTOCOL_TREE_METRIC_C2S_BYTES)).orElse(0).toString()); long receivedBytes = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_S2C_BYTES)).orElse(0).toString()); + results.get(LiveChartConstants.PROTOCOL_TREE_METRIC_S2C_BYTES)).orElse(0).toString()); long sentPackets = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_C2S_PKTS)).orElse(0).toString()); + results.get(LiveChartConstants.PROTOCOL_TREE_METRIC_C2S_PKTS)).orElse(0).toString()); long receivedPackets = Long.parseLong(Optional.ofNullable( - results.get(Constants.PROTOCOL_TREE_METRIC_S2C_PKTS)).orElse(0).toString()); - List<String> protocols = Splitter.on(Constants.ENCAPSULATION_PATH_SEPARATOR).omitEmptyStrings().splitToList(protocolId); + results.get(LiveChartConstants.PROTOCOL_TREE_METRIC_S2C_PKTS)).orElse(0).toString()); + List<String> protocols = Splitter.on(LiveChartConstants.ENCAPSULATION_PATH_SEPARATOR).omitEmptyStrings().splitToList(protocolId); String protocolName = StringUtil.isNotEmpty(protocols) ? protocols.get(protocols.size() - 1) : null; - protocolTree = new ProtocolTree(protocolId, protocolName, null); - protocolTree.setSentBytes(sentBytes); - protocolTree.setReceivedBytes(receivedBytes); - protocolTree.getMetrics().put(Constants.PROTOCOL_TREE_METRIC_SENT_PACKETS, sentPackets); - protocolTree.getMetrics().put(Constants.PROTOCOL_TREE_METRIC_RECEIVED_PACKETS, receivedPackets); - protocolTree.getMetrics().put(Constants.PROTOCOL_TREE_METRIC_SESSIONS, sessions); + protocol = new LiveChartProtocol(protocolId, protocolName, null); + protocol.setSentBytes(sentBytes); + protocol.setReceivedBytes(receivedBytes); + protocol.getMetrics().put(LiveChartConstants.PROTOCOL_TREE_METRIC_SENT_PACKETS, sentPackets); + protocol.getMetrics().put(LiveChartConstants.PROTOCOL_TREE_METRIC_RECEIVED_PACKETS, receivedPackets); + protocol.getMetrics().put(LiveChartConstants.PROTOCOL_TREE_METRIC_SESSIONS, sessions); } catch (BusinessException e) { log.error("Convert Json String to Protocol Tree Object Error ", e); } - return protocolTree; + return protocol; } } diff --git a/src/main/java/com/mesalab/qgw/service/impl/HosServiceImp.java b/src/main/java/com/mesalab/qgw/service/impl/HosServiceImp.java new file mode 100644 index 00000000..480f272d --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/HosServiceImp.java @@ -0,0 +1,250 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.map.MapUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.core.util.XmlUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.HttpConfig; +import com.mesalab.qgw.model.basic.HttpResponseResult; +import com.mesalab.qgw.service.HosService; +import com.mesalab.services.common.entity.KnowledgeConstant; +import com.mesalab.services.configuration.HosConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; + +@Service(value = "hosService") +public class HosServiceImp implements HosService { + + private HosConfig hosConfig; + private HttpConfig httpConfig; + private HttpClientServiceV2 httpClientServiceV2; + private HttpClientService httpClientService; + private static final String TOKEN = "token"; + private static final String CONTENTS = "Contents"; + private static final String USER_DEFINED_META = "UserDefinedMeta"; + private static final String KEY = "Key"; + private static final String LAST_MODIFIED = "LastModified"; + private static final String SIZE = "Size"; + private static final String CHUNK_COUNT = "ChunkCount"; + private static final String REQUEST_ID = "RequestId"; + private static final String MESSAGE = "Message"; + private static final String KEY_COUNT = "KeyCount"; + private static final String FILE_TYPE = "FileType"; + private static final String TASK_ID = "TaskId"; + private static final String RULE_ID = "RuleId"; + + @Override + public BaseResult getFile(String bucketName, String fileName) { + Map<String, String> hosDefaultHeaders = getHosDefaultHeaders(); + String putFileUrl = hosConfig.getUri().concat("/").concat(bucketName).concat("/").concat(fileName); + HttpResponseResult responseResult = httpClientServiceV2.head(putFileUrl, httpConfig.getServerResponseTimeOut(), hosDefaultHeaders); + if (HttpStatusCodeEnum.NOT_FOUND.getCode() == responseResult.getStatusCode()) { + return BaseResultGenerator.success(HttpStatusCodeEnum.NO_CONTENT.getCode(), HttpStatusCodeEnum.NO_CONTENT.getMessage(), null); + } + if (HttpStatusCodeEnum.SUCCESS.getCode() != responseResult.getStatusCode()) { + throw new BusinessException(responseResult.getStatusCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), "Get file failed, message is: " + responseResult.getErrorMessage()); + } + Map<String, String> responseHeaders = responseResult.getResponseHeaders(); + Map<String, Object> result = Maps.newHashMap(); + result.put("file_name", fileName); + result.put("size", StrUtil.isBlankIfStr(responseHeaders.get("File-Size")) ? null : Integer.parseInt(responseHeaders.get("File-Size"))); + result.put("file_url", hosConfig.getUri().concat("/").concat(bucketName).concat("/").concat(fileName)); + result.put("last_modified", responseHeaders.get("Last-Modified")); + return BaseResultGenerator.success(Lists.newArrayList(result)); + } + + @Override + public BaseResult getFileList(String bucketName, String prefix, Map<String, String> udfMetaParam) { + Map<String, String> map = getDataByHos(bucketName, prefix, udfMetaParam); + Map<String, Object> stringObjectMap = XmlUtil.xmlToMap(String.valueOf(map.get("result"))); + if (Integer.parseInt(map.get("status")) == HttpStatusCodeEnum.SUCCESS.getCode()) { + List<Map<String, Object>> dataList = parseXmlData(bucketName, stringObjectMap); + return BaseResultGenerator.success(dataList); + } else { + return BaseResultGenerator.failure(Integer.parseInt(map.get("status")), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), String.valueOf(stringObjectMap.get(REQUEST_ID)), String.valueOf(stringObjectMap.get(MESSAGE))); + } + } + + @Override + public BaseResult uploadFile(String bucketName, String fileName, InputStream inputStream, Map<String, String> headersT) { + Map<String, String> headers = getHosDefaultHeaders(); + if (headersT != null && !headers.isEmpty()) { + headers.putAll(headersT); + } + String putFileUrl = hosConfig.getUri().concat("/").concat(bucketName).concat("/").concat(fileName); + HttpResponseResult responseResult = httpClientServiceV2.put(putFileUrl, inputStream, httpConfig.getServerResponseTimeOut(), headers); + if (HttpStatusCodeEnum.SUCCESS.getCode() != responseResult.getStatusCode()) { + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), "Upload file failed, message is: " + responseResult.getErrorMessage()); + } + return BaseResultGenerator.success(); + } + + @Override + public BaseResult deleteFileList(String bucketName, List<String> fileNames) { + Map<String, String> headers = getHosDefaultHeaders(); + headers.put(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_XML); + String putFileUrl = hosConfig.getUri().concat("/").concat(bucketName).concat("/?delete"); + try { + DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document doc = docBuilder.newDocument(); + + Element deleteElement = doc.createElement("Delete"); + doc.appendChild(deleteElement); + + Element quietElement = doc.createElement("Quiet"); + quietElement.appendChild(doc.createTextNode("true")); + + deleteElement.appendChild(quietElement); + fileNames.forEach(fileName -> deleteElement.appendChild(createElementWithKey(doc, fileName))); + + HttpResponseResult responseResult = httpClientServiceV2.post(putFileUrl, httpConfig.getServerResponseTimeOut(), XmlUtil.toStr(doc, true), headers); + if (HttpStatusCodeEnum.SUCCESS.getCode() != responseResult.getStatusCode()) { + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), "Delete file failed, message is: " + responseResult.getErrorMessage()); + } + } catch (ParserConfigurationException e) { + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), e.getMessage()); + } + return BaseResultGenerator.success(); + } + + private Map<String, String> getHosDefaultHeaders() { + Map<String, String> headers = Maps.newHashMap(); + headers.put(KnowledgeConstant.TOKEN, hosConfig.getToken()); + return headers; + } + + private static Element createElementWithKey(Document doc, String key) { + Element keyElement = doc.createElement("Key"); + keyElement.appendChild(doc.createTextNode(key)); + Element result = doc.createElement("Object"); + result.appendChild(keyElement); + return result; + } + + private List<Map<String, Object>> parseXmlData(String bucketName, Map<String, Object> map) { + List<Map<String, Object>> dataList = Lists.newArrayList(); + int keyCount = Integer.parseInt(String.valueOf(map.get(KEY_COUNT))); + switch (keyCount) { + case 0: + return dataList; + case 1: + dataList = parseSingleValue(bucketName, map); + break; + default: + dataList = parseMultipleValue(bucketName, map); + } + return dataList; + } + + private List<Map<String, Object>> parseMultipleValue(String bucketName, Map<String, Object> map) { + List<Map<String, Object>> dataList = Lists.newArrayList(); + List<Map<String, String>> contentList = (List<Map<String, String>>) map.get(CONTENTS); + if (CollectionUtil.isNotEmpty(contentList)) { + for (Map contents : contentList) { + Map<String, Object> dataMap = buildData(bucketName, contents); + dataList.add(dataMap); + } + } + return dataList; + } + + private List<Map<String, Object>> parseSingleValue(String bucketName, Map<String, Object> map) { + List<Map<String, Object>> dataList = Lists.newArrayList(); + Map<String, String> contentMap = (Map<String, String>) map.get(CONTENTS); + Map<String, Object> stringObjectMap = buildData(bucketName, contentMap); + dataList.add(stringObjectMap); + return dataList; + } + + private Map<String, Object> buildData(String bucketName, Map contents) { + Map<String, Object> dataMap = Maps.newHashMap(); + String key = String.valueOf(contents.get(KEY)); + dataMap.put("file_name", key); + dataMap.put("size", StrUtil.isBlankIfStr(contents.get(SIZE)) ? 0 : Integer.parseInt(String.valueOf(contents.get(SIZE)))); + dataMap.put("file_url", hosConfig.getUri().concat("/").concat(bucketName).concat("/").concat(key)); + dataMap.put("last_modified", String.valueOf(contents.get(LAST_MODIFIED))); + if (MapUtil.isNotEmpty((Map<?, ?>) contents.get(USER_DEFINED_META))) { + Map<String, String> udfMeta = (Map<String, String>) contents.get(USER_DEFINED_META); + Integer chunkCount = StrUtil.isBlankIfStr(udfMeta.get(CHUNK_COUNT)) ? 0 : Integer.parseInt(udfMeta.get(CHUNK_COUNT)); + String taskId = StrUtil.isBlankIfStr(udfMeta.get(TASK_ID)) ? "" : udfMeta.get(TASK_ID); + String ruleId = StrUtil.isBlankIfStr(udfMeta.get(RULE_ID)) ? "" : udfMeta.get(RULE_ID); + dataMap.put("task_id", taskId); + dataMap.put("rule_id", ruleId); + dataMap.put("chunk_count", chunkCount); + } else { + dataMap.put("task_id", ""); + dataMap.put("rule_id", ""); + dataMap.put("chunk_count", 0); + } + return dataMap; + } + + private Map<String, String> getDataByHos(String bucketName, String prefix, Map<String, String> udfMetaParam) { + String url; + if (StrUtil.isNotBlank(prefix) || MapUtil.isNotEmpty(udfMetaParam)) { + url = urlBuilder(bucketName, prefix, udfMetaParam); + } else { + url = hosConfig.getUri().concat("/").concat(bucketName); + } + Map<String, String> result = httpClientService.httpGet(url, getHosDefaultHeaders(), 60000); + return result; + } + + private String urlBuilder(String bucketName, String prefix, Map<String, String> udfMetaParam) { + StringBuffer sb = new StringBuffer(); + sb.append(hosConfig.getUri().concat("/").concat(bucketName)).append("?"); + if (StrUtil.isNotBlank(prefix)) { + sb.append("prefix=").append(prefix).append("&"); + } + if (MapUtil.isNotEmpty(udfMetaParam)) { + for (Map.Entry<String, String> entry : udfMetaParam.entrySet()) { + String paramKey = StrUtil.toSymbolCase(StrUtil.toCamelCase(entry.getKey()), '-'); + String paramValue = StrUtil.toSymbolCase(entry.getValue(), '-'); + sb.append(paramKey).append("=").append(paramValue).append("&"); + } + } + sb.setLength(sb.length() - 1); + return sb.toString(); + } + + @Autowired + private void setHosConfig(HosConfig hosConfig) { + this.hosConfig = hosConfig; + } + + @Autowired + private void setHttpClientService(HttpClientService httpClientService) { + this.httpClientService = httpClientService; + } + + @Autowired + private void setHttpClientServiceV2(HttpClientServiceV2 httpClientServiceV2) { + this.httpClientServiceV2 = httpClientServiceV2; + } + + @Autowired + private void setHttpConfig(HttpConfig httpConfig) { + this.httpConfig = httpConfig; + } +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java b/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java index ccf1dc90..9194d471 100644 --- a/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java +++ b/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java @@ -3,7 +3,7 @@ package com.mesalab.qgw.service.impl; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.google.common.collect.Maps; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; import com.mesalab.qgw.model.basic.HttpConfig; import com.geedgenetworks.utils.StringUtil; @@ -45,6 +45,7 @@ import java.util.HashMap; import java.util.Map; @Service +@Deprecated public class HttpClientService { private static final Log log = LogFactory.get(); @@ -223,19 +224,19 @@ public class HttpClientService { resultMap.put("result", EntityUtils.toString(entity, "UTF-8")); } catch (ClientProtocolException e) { log.error("协议错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVICE_UNAVAILABLE.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVICE_UNAVAILABLE.getCode())); resultMap.put("message", e.getMessage()); } catch (ParseException e) { log.error("解析错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVICE_UNAVAILABLE.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVICE_UNAVAILABLE.getCode())); resultMap.put("message", e.getMessage()); } catch (IOException e) { log.error("IO错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.BAD_GATEWAY.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.BAD_GATEWAY.getCode())); resultMap.put("message", e.getMessage()); } catch (Exception e) { log.error("其它错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVER_ERROR.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVER_ERROR.getCode())); resultMap.put("message", e.getMessage()); } finally { if (null != response) { @@ -271,19 +272,19 @@ public class HttpClientService { resultMap.put("result", EntityUtils.toString(entity, "UTF-8")); } catch (ClientProtocolException e) { log.error("ClientProtocolException:{}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVICE_UNAVAILABLE.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVICE_UNAVAILABLE.getCode())); resultMap.put("message", e.getMessage()); } catch (ParseException e) { log.error("ParseException:{}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVICE_UNAVAILABLE.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVICE_UNAVAILABLE.getCode())); resultMap.put("message", e.getMessage()); } catch (IOException e) { log.error("IOException:{}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.BAD_GATEWAY.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.BAD_GATEWAY.getCode())); resultMap.put("message", e.getMessage()); } catch (Exception e) { log.error("Exception:{}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVER_ERROR.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVER_ERROR.getCode())); resultMap.put("message", e.getMessage()); } finally { if (null != response) { @@ -323,19 +324,19 @@ public class HttpClientService { resultMap.put("result", EntityUtils.toString(entity, "UTF-8")); } catch (ClientProtocolException e) { log.error("ClientProtocolException:{}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVICE_UNAVAILABLE.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVICE_UNAVAILABLE.getCode())); resultMap.put("message", e.getMessage()); } catch (ParseException e) { log.error("ParseException:{}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVICE_UNAVAILABLE.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVICE_UNAVAILABLE.getCode())); resultMap.put("message", e.getMessage()); } catch (IOException e) { log.error("IOException:{}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.BAD_GATEWAY.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.BAD_GATEWAY.getCode())); resultMap.put("message", e.getMessage()); } catch (Exception e) { log.error("Exception:{}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVER_ERROR.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVER_ERROR.getCode())); resultMap.put("message", e.getMessage()); } finally { if (null != response) { @@ -372,19 +373,19 @@ public class HttpClientService { resultMap.put("result", EntityUtils.toString(entity, "UTF-8")); } catch (ClientProtocolException e) { log.error("ClientProtocolException: {}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVICE_UNAVAILABLE.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVICE_UNAVAILABLE.getCode())); resultMap.put("message", e.getMessage()); } catch (ParseException e) { log.error("ParseException: {}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVICE_UNAVAILABLE.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVICE_UNAVAILABLE.getCode())); resultMap.put("message", e.getMessage()); } catch (IOException e) { log.error("IOException: {}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.BAD_GATEWAY.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.BAD_GATEWAY.getCode())); resultMap.put("message", e.getMessage()); } catch (Exception e) { log.error("Exception: {}", e.getMessage()); - resultMap.put("status", String.valueOf(ResultStatusEnum.SERVER_ERROR.getCode())); + resultMap.put("status", String.valueOf(HttpStatusCodeEnum.SERVER_ERROR.getCode())); resultMap.put("message", e.getMessage()); } finally { if (null != response) { @@ -457,7 +458,7 @@ public class HttpClientService { HttpEntity entity = response.getEntity(); // 获取响应信息 msg = EntityUtils.toString(entity, "UTF-8"); - if (statusCode != ResultStatusEnum.SUCCESS.getCode()) { + if (statusCode != HttpStatusCodeEnum.SUCCESS.getCode()) { throw new BusinessException("Http get content is :" + msg); } } catch (URISyntaxException e) { @@ -512,7 +513,7 @@ public class HttpClientService { HttpEntity entity = response.getEntity(); // 获取响应信息 msg = EntityUtils.toString(entity, "UTF-8"); - if (statusCode != ResultStatusEnum.SUCCESS.getCode() && statusCode != ResultStatusEnum.CREATED.getCode()) { + if (statusCode != HttpStatusCodeEnum.SUCCESS.getCode() && statusCode != HttpStatusCodeEnum.CREATED.getCode()) { throw new BusinessException(msg); } } catch (URISyntaxException e) { @@ -596,7 +597,7 @@ public class HttpClientService { // 执行请求 response = httpClient.execute(httpGet); int code = response.getStatusLine().getStatusCode(); - if (code != ResultStatusEnum.SUCCESS.getCode()) { + if (code != HttpStatusCodeEnum.SUCCESS.getCode()) { log.error("current file: {}, get InputStream error, status code:{}, entity: {}", url, code, EntityUtils.toString(response.getEntity())); } else { // 获取响应实体 @@ -623,8 +624,8 @@ public class HttpClientService { } } - return result; } + return result; } /** diff --git a/src/main/java/com/mesalab/qgw/service/impl/HttpClientServiceV2.java b/src/main/java/com/mesalab/qgw/service/impl/HttpClientServiceV2.java new file mode 100644 index 00000000..5ff13fc1 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/HttpClientServiceV2.java @@ -0,0 +1,366 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.exceptions.ExceptionUtil; +import com.geedgenetworks.utils.StringUtil; +import com.google.common.collect.Maps; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.HttpConfig; +import com.mesalab.qgw.model.basic.HttpResponseResult; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.apache.http.*; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.*; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.conn.ConnectionKeepAliveStrategy; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicHeaderElementIterator; +import org.apache.http.protocol.HTTP; +import org.apache.http.util.EntityUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; +import javax.net.ssl.*; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.X509Certificate; +import java.util.HashMap; +import java.util.Map; +@Service +@Slf4j +public class HttpClientServiceV2 { + private HttpConfig httpConfig; + private PoolingHttpClientConnectionManager connectionManager; + private static final String CONTENT_TYPE = "Content-Type"; + private static final String APPLICATION_JSON = "application/json"; + + /** + * Init PoolingHttpClientConnectionManager, support http and https + */ + @PostConstruct + private void initConnectionManager() { + try { + // Use TrustManager to trust all certificates + X509TrustManager trustManager = new X509TrustManager() { + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + @Override + public void checkClientTrusted(X509Certificate[] xcs, String str) { + } + @Override + public void checkServerTrusted(X509Certificate[] xcs, String str) { + } + }; + SSLContext ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); + ctx.init(null, new TrustManager[]{trustManager}, null); + SSLConnectionSocketFactory socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE); + Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", socketFactory).build(); + + if (connectionManager == null) { + connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connectionManager.setMaxTotal(httpConfig.getMaxConnectionNum()); + connectionManager.setDefaultMaxPerRoute(httpConfig.getMaxPerRoute()); + } + log.info("Initializing PoolingHttpClientConnectionManager Complete"); + } catch (KeyManagementException e) { + log.error("KeyManagementException", e); + throw new BusinessException(e.getMessage()); + } catch (NoSuchAlgorithmException e) { + log.error("NoSuchAlgorithmException", e); + throw new BusinessException(e.getMessage()); + } + } + + @Autowired + public void setHttpConfig(@Qualifier("httpConfig") HttpConfig httpConfig) { + this.httpConfig = httpConfig; + } + + + /** + * Specify the timeout time for each request and returned the corresponding CloseableHttpClient + * @param socketTimeOut + * @return CloseableHttpClient instance + */ + private CloseableHttpClient getHttpClient(int socketTimeOut) { + RequestConfig requestConfig = RequestConfig.custom() + .setConnectionRequestTimeout(httpConfig.getServerRequestTimeOut()) + .setConnectTimeout(httpConfig.getServerConnectTimeOut()) + .setSocketTimeout(socketTimeOut) // Setting the socket timeout time + .build(); + //Http request for retry mechanism. if return false, no retry, otherwise retry. + HttpRequestRetryHandler retry = (exception, executionCount, context) -> { + if (executionCount >= 3) { // Maximum number of retries + return false; + } + // If the server has not responded, retry + if (exception instanceof NoHttpResponseException) { + return true; + } + //no retry for SSLHandshakeException + if (exception instanceof SSLHandshakeException) { + return false; + } + if(exception instanceof SocketTimeoutException) { + if (exception.getMessage().contains("Read timed out")) { + return false; + } + } + //No retry for UnknownHostException, indicates the host is unreachable + if (exception instanceof UnknownHostException) { + return false; + } + // No retry for ConnectTimeoutException, indicates the connection is refused + if (exception instanceof ConnectTimeoutException) { + return false; + } + // No retry for SSLException, indicates the SSL handshake error + if (exception instanceof SSLException) { + return false; + } + // If the request has been sent, but the client has been interrupted, retry + if (exception instanceof InterruptedIOException) { + return true; + } + + HttpClientContext clientContext = HttpClientContext.adapt(context); + HttpRequest request = clientContext.getRequest(); + // If the request is idempotent, retry + if (!(request instanceof HttpEntityEnclosingRequest)) { + return true; + } + return false; + }; + + + ConnectionKeepAliveStrategy myStrategy = (response, context) -> { + HeaderElementIterator it = new BasicHeaderElementIterator + (response.headerIterator(HTTP.CONN_KEEP_ALIVE)); + while (it.hasNext()) { + HeaderElement headerElement = it.nextElement(); + String param = headerElement.getName(); + String value = headerElement.getValue(); + if (value != null && param.equalsIgnoreCase("timeout")) { + return Long.parseLong(value) * 1000; + } + } + return 60 * 1000; // if not specified, default keep alive is 60 seconds + }; + + //Create a CloseableHttpClient instance + return HttpClients.custom() + // Set the timeout time for each request + .setDefaultRequestConfig(requestConfig) + // Set the retry mechanism + .setRetryHandler(retry) + .setKeepAliveStrategy(myStrategy) + // set the connection pool + .setConnectionManager(connectionManager) + .build(); + } + public HttpResponseResult get(String url) { + return get(url, httpConfig.getServerResponseTimeOut(), null); + } + public HttpResponseResult get(String url, int socketTimeout) { + return get(url, socketTimeout, null); + } + + public HttpResponseResult get(String url, int socketTimeout, Map<String, String> headers) { + return executeHttpRequest(new HttpGet(url), socketTimeout, headers); + } + + public HttpResponseResult head(String url, int socketTimeout) { + return executeHttpRequest(new HttpHead(url), socketTimeout, null); + } + + public HttpResponseResult head(String url, int socketTimeout, Map<String, String> headers) { + return executeHttpRequest(new HttpHead(url), socketTimeout, headers); + } + + public HttpResponseResult post(String url, int socketTimeout, String requestBody, Map<String, String> headers) { + if (StringUtil.isEmpty(headers)) { + headers = Maps.newHashMap(); + } + if(headers.get(CONTENT_TYPE) == null) { + headers.put(CONTENT_TYPE, APPLICATION_JSON); + } + HttpPost httpPost = new HttpPost(url); + httpPost.setEntity(new ByteArrayEntity(requestBody.getBytes(StandardCharsets.UTF_8))); + return executeHttpRequest(httpPost, socketTimeout, headers); + } + + public HttpResponseResult post(String url, int socketTimeout, String requestBody){ + return post(url, socketTimeout, requestBody , null); + } + + public HttpResponseResult post(String url, String requestBody, Map<String, String> headers){ + return post(url, httpConfig.getServerResponseTimeOut(), requestBody , headers); + } + + public HttpResponseResult post(String url, String requestBody){ + return post(url, httpConfig.getServerResponseTimeOut(), requestBody , null); + } + + public HttpResponseResult delete(String url) { + return executeHttpRequest(new HttpDelete(url), httpConfig.getServerResponseTimeOut(), null); + } + + public HttpResponseResult delete(String url, int socketTimeout) { + return executeHttpRequest(new HttpDelete(url), socketTimeout, null); + } + + + public HttpResponseResult delete(String url, int socketTimeout, Map<String, String> headers){ + return executeHttpRequest(new HttpDelete(url), socketTimeout, headers); + } + + + public HttpResponseResult put(String url, InputStream in, int socketTimeout, Map<String, String> headers) { + HttpPut httpPut = new HttpPut(url); + if (StringUtil.isEmpty(headers)) { + headers = Maps.newHashMap(); + } + if(headers.get(CONTENT_TYPE) == null) { + headers.put(CONTENT_TYPE, APPLICATION_JSON); + } + if (in != null) { + httpPut.setEntity(new InputStreamEntity(in)); + } + return executeHttpRequest(httpPut, socketTimeout, headers); + } + + public Map<String, String> getHttpPostResponseHeader(String url, int socketTimeOut, Map<String, String> headers) { + CloseableHttpResponse response = null; + HashMap<String, String> map = Maps.newHashMap(); + try { + HttpPost httpPost = new HttpPost(url); + headers.forEach(httpPost::setHeader); + response = getHttpClient(socketTimeOut).execute(httpPost); + Header[] Headers = response.getAllHeaders(); + for (Header h : Headers) { + map.put(h.getName().toUpperCase(), h.getValue()); + } + } catch (Exception e) { + throw new BusinessException(handleException(e), CommonErrorCode.HTTP_REQUEST_EXCEPTION.getCode(), e.getMessage()); + } finally { + closeResponse(response); + } + return map; + } + + public InputStream getInputStream(String url, int socketTimeout, Map<String, String> headers) { + CloseableHttpResponse response = null; + InputStream result = null; + try { + HttpGet httpGet = new HttpGet(url); + if (headers != null) { + headers.forEach(httpGet::setHeader); + } + response = getHttpClient(socketTimeout).execute(httpGet); + result = IOUtils.toBufferedInputStream(response.getEntity().getContent()); + } catch (Exception e) { + throw new BusinessException(handleException(e), CommonErrorCode.HTTP_REQUEST_EXCEPTION.getCode(), e.getMessage()); + } finally { + closeResponse(response); + } + return result; + } + + + private HttpResponseResult executeHttpRequest(HttpRequestBase request, int socketTimeout, Map<String, String> headers) { + HttpResponseResult responseResult = new HttpResponseResult(); + CloseableHttpResponse response = null; + try { + if (headers != null) { + headers.forEach(request::setHeader); + } + response = getHttpClient(socketTimeout).execute(request); + HttpEntity entity = response.getEntity(); + responseResult.setStatusCode(response.getStatusLine().getStatusCode()); + if (entity != null) { + responseResult.setResponseBody(EntityUtils.toString(entity, StandardCharsets.UTF_8)); + } + if (response.getAllHeaders() != null) { + Map<String, String> headersMap = Maps.newHashMap(); + for (Header header : response.getAllHeaders()) { + headersMap.put(header.getName(), header.getValue()); + } + responseResult.setResponseHeaders(headersMap); + } + if (responseResult.getStatusCode() >= 300) { + log.error("Http request failed, url:{}, statusCode:{}, responseBody:{}", + request.getURI(), responseResult.getStatusCode(), responseResult.getResponseBody()); + responseResult.setErrorMessage(responseResult.getResponseBody()); + } + } catch (Exception e) { + responseResult.setStatusCode(handleException(e)); + responseResult.setErrorMessage(ExceptionUtil.getRootCauseMessage(e)); + } finally { + closeResponse(response); + } + return responseResult; + } + + private int handleException(Exception e) { + if (e instanceof ClientProtocolException || e instanceof ParseException) { + log.error("Protocol or Parsing error:{}", e.getMessage()); + return HttpStatusCodeEnum.SERVICE_UNAVAILABLE.getCode(); + } else if (e instanceof SocketTimeoutException) { + log.error("Socket timeout error:{}", e.getMessage()); + return HttpStatusCodeEnum.GATEWAY_TIMEOUT.getCode(); + } else if (e instanceof ConnectTimeoutException) { + log.error("Connect timeout error:{}", e.getMessage()); + return HttpStatusCodeEnum.GATEWAY_TIMEOUT.getCode(); + } else if (e instanceof UnknownHostException) { + log.error("Unknown host error:{}", e.getMessage()); + return HttpStatusCodeEnum.BAD_GATEWAY.getCode(); + } else if (e instanceof InterruptedIOException) { + log.error("Interrupted error:{}", e.getMessage()); + return HttpStatusCodeEnum.BAD_GATEWAY.getCode(); + } else if (e instanceof IOException) { + log.error("IO error:{}", e.getMessage()); + return HttpStatusCodeEnum.BAD_GATEWAY.getCode(); + } else { + log.error("Other error:{}", e.getMessage()); + return HttpStatusCodeEnum.SERVER_ERROR.getCode(); + } + } + + private void closeResponse(CloseableHttpResponse response) { + if (response != null) { + try { + EntityUtils.consumeQuietly(response.getEntity()); + response.close(); + } catch (IOException e) { + log.error("Close Connection Exception: {}", e.getMessage()); + } + } + } + +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java deleted file mode 100644 index 5eff14b3..00000000 --- a/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java +++ /dev/null @@ -1,575 +0,0 @@ -package com.mesalab.qgw.service.impl; - -import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.util.NumberUtil; -import cn.hutool.core.util.StrUtil; -import cn.hutool.json.JSONException; -import cn.hutool.json.JSONUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONWriter; -import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.config.annotation.NacosValue; -import com.alibaba.nacos.api.config.listener.AbstractListener; -import com.alibaba.nacos.api.exception.NacosException; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.jayway.jsonpath.JsonPath; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.common.enums.MetadataTypeEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.nacos.NacosConfig; -import com.mesalab.common.nacos.NacosConst; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.model.basic.ClickHouseHttpSource; -import com.mesalab.qgw.model.metadata.MetadataBean; -import com.mesalab.qgw.model.metadata.MetadataConfig; -import com.mesalab.qgw.service.QueryService; -import com.mesalab.qgw.service.MetadataService; -import com.geedgenetworks.utils.StringUtil; -import org.apache.avro.Schema; -import org.apache.avro.SchemaParseException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.util.*; - -/** - * @Date: 2021-03-11 15:48 - * @Author : liuyongqiang - * @ClassName : NacosMetadataServiceImpl - * @Description : 获取Nacos注册中心元数据实现类 - */ -@Service("metadataService") -public class MetadataServiceImpl implements MetadataService { - - private static final Log log = LogFactory.get(); - @Autowired - private NacosConfig nacosConfig; - @Autowired - private ConfigService systemConfigService; - @Autowired - private ConfigService pubConfigService; - @Autowired - private ClickHouseHttpSource clickHouseHttpSource; - @Autowired - private QueryService queryService; - @Autowired - public MetadataConfig metadataConfig; - @Autowired - Environment environment; - @NacosValue(value = "${switch.version.schema}", autoRefreshed = true) - private String schemaVersion; - - private Map<String, Schema> schemaCache = Maps.newHashMap(); - private Map<String, Object> originalCache = Maps.newHashMap(); - private final static String KEY_REF = "$ref"; - private final static String KEY = "key"; - private final static String VALUE = "value"; - private final static String CODE = "code"; - private final static String INDEX_KEY = "index_key"; - - @PostConstruct - public void init() { - addMetadataListener(); - log.info("Initializing Schema Tables Complete"); - } - - @Override - public Map<String, Object> getSchemaInfo(String type, String name, boolean displayStorageSize) { - if (MetadataTypeEnum.TABLES.getValue().equals(type)) { - return loadTables(name); - } else if (MetadataTypeEnum.FIELDS.getValue().equals(type)) { - return loadFields(name, displayStorageSize); - } - throw new QGWBusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.SCHEMA_URL_NOT_FOUND)); - } - - @Override - public String getPartitionKey(String tableName) { - String partitionKey = StringUtil.EMPTY; - Schema schema = getSchemaByName(tableName); - Map doc = (Map) JSON.parseObject(schema.getDoc(), Map.class); - if (StringUtil.isNotEmpty(doc)) { - Object partition_key = doc.get("partition_key"); - partitionKey = StringUtil.isEmpty(partition_key) ? StringUtil.EMPTY : partition_key.toString(); - } - return partitionKey; - } - - @Override - public List<String> getIndexKey(String tableName) { - Map schemaJsonMap = loadFields(tableName, false); - if (StringUtil.isEmpty(schemaJsonMap) || StringUtil.isEmpty(schemaJsonMap.get("doc"))) { - return Lists.newArrayList(); - } - Map doc = (Map) schemaJsonMap.get("doc"); - Object indexKey = doc.get(INDEX_KEY); - return StringUtil.isEmpty(indexKey) ? Lists.newArrayList() : (List<String>) indexKey; - } - - @Override - public String getValueByKeyInSchemaDoc(String tableName, String key) { - Schema schema = getSchemaByName(tableName); - Map doc = (Map) JSON.parseObject(schema.getDoc(), Map.class); - if (StringUtil.isNotEmpty(doc)) { - Object value = doc.get(key); - if (value instanceof Map) { - return JSONUtil.toJsonStr(JSONUtil.parseObj(value)); - } else { - return StringUtil.isEmpty(value) ? StringUtil.EMPTY : value.toString(); - } - } - return StringUtil.EMPTY; - } - - @Override - public String getDBTypeByTableName(String tableName) { - Schema schema = getSchemaByName(tableName); - String data = StringUtil.EMPTY; - if (StringUtil.isEmpty(schema)) { - return data; - } - for (MetadataBean meta : metadataConfig.getMetadata()) { - if (meta.getTables().contains(tableName)) { - return getDBTypeByGroup(meta.getGroup()); - } - } - return data; - } - - private String getDBTypeByGroup(String group) { - String type; - switch (group) { - case "CLICKHOUSE_GROUP": - type = DBTypeEnum.CLICKHOUSE.getValue(); - break; - case "HBASE_GROUP": - type = DBTypeEnum.HBASE.getValue(); - break; - case "DRUID_GROUP": - type = DBTypeEnum.DRUID.getValue(); - break; - default: - type = StringUtil.EMPTY; - } - return type; - } - - @Override - public String getDBNameByTableName(String tableName) { - Schema schema = getSchemaByName(tableName); - return StringUtil.isNotEmpty(schema) ? schema.getNamespace() : clickHouseHttpSource.getDbName(); - } - - @Override - public List<String> getAllTable() { - List<String> list = new ArrayList<>(); - metadataConfig.getMetadata().forEach(o -> { - if (StringUtil.isNotEmpty(o.getTables())) { - list.addAll(o.getTables()); - } - }); - return list; - } - - @Override - public Schema getSchemaByName(String tableName) { - if (!getAllTable().contains(tableName)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), - ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), - tableName + ":"+QGWMessageConst.TABLE_NOT_EXIST)); - } - Schema schema; - if (!StrUtil.isBlankIfStr(schema = schemaCache.get(tableName))) { - return schema; - } - try { - String content = systemConfigService.getConfig(tableName.concat(NacosConst.JSON_SUFFIX), nacosConfig.getGroup(), 3000); - if (StrUtil.isBlankIfStr(content)) { - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),tableName + ":"+QGWMessageConst.QUERY_SCHEMA_ERROR)); - } - schemaCache.put(tableName, schema = parseSchema(content)); - systemConfigService.addListener(tableName.concat(NacosConst.JSON_SUFFIX), nacosConfig.getGroup(), new AbstractListener() { - @Override - public void receiveConfigInfo(String configInfo) { - log.info("ReceiveConfigInfo Schema Fields {}", configInfo); - schemaCache.remove(tableName); - schemaCache.put(tableName, parseSchema(configInfo)); - } - }); - } catch (NacosException e) { - log.error("NacosException:{}", e); - } - return schema; - } - - @Override - public BaseResult updateSchema(String name, Map<String, Object> paramMap) { - try { - String content = systemConfigService.getConfig(name.concat(".json"), nacosConfig.getGroup(), 3000); - updateSchema(name, paramMap, content); - String indexTable = getValueByKeyInSchemaDoc(name, "index_table"); - if (StringUtil.isNotBlank(indexTable)) { - List<String> list = Splitter.on(",").omitEmptyStrings().splitToList(indexTable); - for (String item : list) { - String config = systemConfigService.getConfig(item.concat(".json"), nacosConfig.getGroup(), 3000); - updateSchema(item, paramMap, config); - } - } - return BaseResultGenerator.success4Message("ok"); - } catch (NacosException e) { - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - e.getMessage()); - } - } - - private boolean updateSchema(String name, Map<String, Object> paramMap, String content) throws NacosException { - if (StringUtil.isNotEmpty(content)) { - Map<String, Object> data = JSON.parseObject(content, Map.class); - data.put("doc", rewriteDoc("ttl", data.get("doc"), paramMap.get("doc"))); - List<Map<String, Object>> fields = (List<Map<String, Object>>) data.get("fields"); - List<String> indexKey = getIndexKey(name); - for (Map<String, Object> field : fields) { - if (indexKey.contains(field.get("name"))) { - Map<String, Object> ttlNull = Maps.newHashMap(); - ttlNull.put("ttl", null); - field.put("doc", rewriteDoc("ttl", field.get("doc"), ttlNull)); - } else { - field.put("doc", rewriteDoc("ttl", field.get("doc"), JsonPath.read(paramMap, "$.fields[?(@.name == \"" + field.get("name") + "\")].doc"))); - } - field.put("doc", rewriteDoc("visibility", field.get("doc"), JsonPath.read(paramMap, "$.fields[?(@.name == \"" + field.get("name") + "\")].doc"))); - } - content = JSON.toJSONString(data, JSONWriter.Feature.WriteNulls); - log.info("push Schema, content is: {}", content); - return systemConfigService.publishConfig(name.concat(".json"), nacosConfig.getGroup(), JSONUtil.formatJsonStr(content)); - } else { - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),QGWMessageConst.QUERY_SCHEMA_ERROR)); - } - } - - /** - * Desc: 重写[tables |field].doc - * @param key - * @param original - * @param param - * @return {@link Map< String, Object>} - * @created by wWei - * @date 2022/5/17 10:02 上午 - */ - private Map<String, Object> rewriteDoc(String key, Object original, Object param) { - Map<String, Object> doc = Maps.newHashMap(); - if (StringUtil.isNotEmpty(original)) { - doc = (Map<String, Object>) original; - } - Map<String, Object> paramDoc = Maps.newHashMap(); - if (param instanceof Map) { - paramDoc = (Map<String, Object>) param; - } else if (param instanceof List) { - List item = (List) param; - paramDoc = (Map<String, Object>) item.get(0); - } - if (paramDoc.containsKey(key)) { - Object value = paramDoc.get(key); - if (NumberUtil.isNumber(String.valueOf(value))) { - doc.put(key, Long.parseLong(value.toString())); - } else { - doc.put(key, value); - } - } else { - doc.remove(key); - } - return doc; - } - - private void addMetadataListener() { - try { - systemConfigService.addListener(NacosConst.META_DATA_ID, nacosConfig.getGroup(), new AbstractListener() { - @Override - public void receiveConfigInfo(String configInfo) { - log.info("ReceiveConfigInfo metadata {}", configInfo); - schemaCache = Maps.newHashMap(); - } - }); - } catch (NacosException e) { - log.error("NacosException: ", e); - } - } - - private Map loadFields(String name, boolean displayStorageSize) { - Schema schema = getSchemaByName(name); - if (StringUtil.isEmpty(schema)) { - throw new QGWBusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), QGWMessageConst.SCHEMA_URL_NOT_FOUND); - } - LinkedHashMap resultMap = JSON.parseObject(String.valueOf(schema), LinkedHashMap.class); - LinkedHashMap schemaDoc = JSON.parseObject(schema.getDoc(), LinkedHashMap.class); - if (StringUtil.isEmpty(schemaDoc) && - StringUtil.isNotEmpty(schema.getDoc())) { - log.error("{} schema's doc isn't jsonString and won't convert: {}", name, schema.getDoc()); - } - List<Map> data = Lists.newArrayList(); - if (displayStorageSize) { - data = getStorageSize(name); - schemaDoc = (LinkedHashMap) fillSchemaStorageSize(schemaDoc, data); - } - if (StringUtil.isNotEmpty(schemaDoc)) { - resultMap.put("doc", schemaDoc); - } - List<Map> fields = (List<Map>) resultMap.get("fields"); - LinkedHashMap mapDoc; - for (Map next : fields) { - mapDoc = JSON.parseObject((String) next.get("doc"), LinkedHashMap.class); - mapDoc = (LinkedHashMap) fillFieldStorageSize(data, mapDoc, next); - if (StringUtil.isNotEmpty(mapDoc)) { - next.put("doc", mapDoc); - } - if (StringUtil.isEmpty(mapDoc) && !StringUtil.isEmpty(next.get("doc"))) { - log.error("{} field's doc isn't jsonString and won't convert: {}", next.get("name"), next.get("doc")); - } - } - fillReference(resultMap, name); - return resultMap; - } - - private Map fillSchemaStorageSize(LinkedHashMap schemaDoc, List<Map> data) { - if (CollectionUtil.isEmpty(data)) { - return schemaDoc; - } - if (StringUtil.isEmpty(schemaDoc)) { - schemaDoc = new LinkedHashMap(); - } - long total = 0; - for (Map datum : data) { - total += Long.parseLong(String.valueOf(datum.get("bytes"))); - } - schemaDoc.put("size", total); - return schemaDoc; - } - - private Map fillFieldStorageSize(List<Map> data, LinkedHashMap mapDoc, Map next) { - if (CollectionUtil.isEmpty(data)) { - return mapDoc; - } - if (StringUtil.isEmpty(mapDoc)) { - mapDoc = new LinkedHashMap(); - } - try { - List<Object> name = JsonPath.read(data, "$.[?(@.field == \"" + next.get("name") + "\")].bytes"); - mapDoc.put("size", Long.parseLong(String.valueOf(name.get(0)))); - } catch (RuntimeException e) { - log.error("fill {} field storage size error, message is: {}", next.get("name"), e.getMessage()); - } - return mapDoc; - } - - private List<Map> getStorageSize(String name) { - try { - List<String> symbols = (List<String>) loadTables(clickHouseHttpSource.getDbName()).get("symbols"); - String sql = null; - if (symbols.contains(name)) { - sql = String.format(Objects.requireNonNull(environment.getProperty("SCHEMA_STORAGE_SIZE")), name, clickHouseHttpSource.getDbName()); - } - BaseResult baseResult = StringUtil.isBlank(sql) ? BaseResultGenerator.success(Lists.newArrayList()) : queryService.executeQuery(QueryProfile.builder().query(sql).build()); - if (baseResult.isSuccess()) { - return (List<Map>) baseResult.getData(); - } else { - log.error("get {} schema storage size error on DB.", name); - } - } catch (RuntimeException ex) { - log.error("get {} schema storage size error on DB, message is {}", name, ex.getMessage()); - } - return Lists.newArrayList(); - } - - /** - * @param map 需操作对象 - * @param cfgName 当前文件名称 - * @return void - * @Description $ref实际引用部分赋值 - * @author wanghao - * @date 2021/9/1 17:02 - */ - - private void fillReference(Map map, String cfgName) { - for (Object mapKey : map.keySet()) { - Object keyObj = map.get(mapKey); - if (keyObj instanceof Map) { - Map mapValue = (Map) keyObj; - if (mapValue.containsKey(KEY_REF)) { - if (INDEX_KEY.equals(String.valueOf(mapKey))) { - String s = String.valueOf(mapValue.get(KEY_REF)); - mapValue.put(KEY_REF, s.contains("public_schema_info_2311.json") ? s.concat("/" + schemaVersion) : s); - } - Object refValue = getRefValue(cfgName, mapValue); - map.put(mapKey, regexValue(refValue, mapValue)); - } else { - fillReference(mapValue, cfgName); - } - } else if (keyObj instanceof List) { - List listValue = (List) keyObj; - for (Object next : listValue) { - if (next instanceof Map) { - fillReference((Map) next, cfgName); - } - } - } - } - } - - @Override - public Object getCfg(String cfgName) { - if (originalCache.containsKey(cfgName)) { - return originalCache.get(cfgName); - } - String content = null; - try { - content = getPubCfg(cfgName); - if (!StrUtil.isBlankIfStr(content)) { - originalCache.put(cfgName, content); - return content; - } - content = systemConfigService.getConfig(cfgName, nacosConfig.getGroup(), 3000); - originalCache.put(cfgName, content); - systemConfigService.addListener(cfgName, nacosConfig.getGroup(), new AbstractListener() { - @Override - public void receiveConfigInfo(String configInfo) { - log.info("ReceiveConfigInfo Schema Fields {}", configInfo); - originalCache.put(cfgName, configInfo); - } - }); - } catch (NacosException e) { - log.error("NacosException:{}", e); - } - return content; - } - - private String getPubCfg(String cfgName) throws NacosException { - String content = pubConfigService.getConfig(cfgName, NacosConst.DEFAULT_GROUP, 3000); - pubConfigService.addListener(cfgName, NacosConst.DEFAULT_GROUP, new AbstractListener() { - @Override - public void receiveConfigInfo(String configInfo) { - log.info("ReceiveConfigInfo Schema Fields {}", configInfo); - originalCache.put(cfgName, configInfo); - } - }); - return content; - } - - private Object getRefValue(String schemaName, Map map) { - Object data = null; - String[] split = getRefStrArray(map); - String jsonPath = getJsonPath(split); - try { - Object cfg = getCfg(getFileName(split, schemaName)); - data = JsonPath.read(String.valueOf(cfg), jsonPath); - } catch (RuntimeException e) { - log.warn("read reference schema error: {}", e); - } - return data; - } - - private Object regexValue(Object obj, Map<String, String> map) { - if (map.size() <= 1) { - return obj; - } - List<Object> key = Lists.newArrayList(); - List<Object> value = Lists.newArrayList(); - try { - key.addAll(JsonPath.read(obj, map.get(KEY))); - value.addAll(JsonPath.read(obj, map.get(VALUE))); - } catch (RuntimeException ex) { - log.warn("parse reference schema error: JsonPath.read error"); - return Lists.newArrayList(); - } - if (key.size() != value.size()) { - log.warn("parse reference schema error: key and value non-correspondence"); - return Lists.newArrayList(); - } - List<Map<String, Object>> list = Lists.newArrayList(); - for (int i = 0; i < key.size(); i++) { - HashMap<String, Object> item = Maps.newLinkedHashMap(); - item.put(CODE, key.get(i)); - item.put(VALUE, value.get(i)); - list.add(item); - } - return list; - } - - private String[] getRefStrArray(Map refMap) { - String v = (String) refMap.get(KEY_REF); - return v.split("#", 2); - } - - private String getJsonPath(String[] strArray) { - return "$".concat(strArray[1].replace("/", ".")); - } - - private String getFileName(String[] strArray, String fileName) { - return StrUtil.isEmpty(strArray[0]) ? fileName.concat(NacosConst.JSON_SUFFIX) : strArray[0]; - } - - /** - * @param name - * @Description: 从Nacos配置中心获取Tables - * @Author: liuyongqiang - * @Date: 2021/3/11 18:17 - * @return: com.mesalab.common.base.BaseResult - **/ - private Map<String, Object> loadTables(String name) { - Map<String, Object> date = new LinkedHashMap<>(); - List<String> tables = new ArrayList<>(); - metadataConfig.getMetadata().forEach(o -> { - if (o.getNamespace().equalsIgnoreCase(name)) { - tables.addAll(o.getTables()); - } - } - ); - if (tables.isEmpty()) { - throw new QGWBusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), QGWMessageConst.SCHEMA_URL_NOT_FOUND); - } - date.put("type", "enum"); - date.put("name", name); - date.put("symbols", tables); - return date; - } - - private Schema parseSchema(String configInfo) { - if (StrUtil.isBlank(configInfo)) return null; - Map schemaMap = null; - try { - schemaMap = JSON.parseObject(configInfo, Map.class); - Object docSchema; - if (StringUtil.isNotEmpty(docSchema = schemaMap.get("doc"))) { - schemaMap.put("doc", JSON.toJSONString(docSchema, JSONWriter.Feature.WriteNulls)); - } - Object fieldsObj; - if (StringUtil.isNotEmpty(fieldsObj = schemaMap.get("fields"))) { - List<Map> fields = (List<Map>) fieldsObj; - for (Map next : fields) { - Object docField; - if (StringUtil.isEmpty(docField = next.get("doc"))) { - continue; - } - next.put("doc", JSON.toJSONString(docField, JSONWriter.Feature.WriteNulls)); - } - } - } catch (JSONException | SchemaParseException | ClassCastException e) { - log.error("update Schema error: {}", e); - } - return new Schema.Parser().parse(JSON.toJSONString(schemaMap, JSONWriter.Feature.WriteNulls)); - } -} diff --git a/src/main/java/com/mesalab/qgw/service/impl/PacketCombineDslServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/PacketCombineDslServiceImpl.java new file mode 100644 index 00000000..d7d178ad --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/PacketCombineDslServiceImpl.java @@ -0,0 +1,268 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.util.ArrayUtil; +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.google.common.collect.Maps; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.enums.QueryOption; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.qgw.model.basic.DSLQueryContext; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import com.mesalab.qgw.model.basic.EngineConfigSource; +import com.mesalab.qgw.model.basic.SQLQueryContext; +import com.mesalab.qgw.service.*; +import com.mesalab.services.common.property.SqlPropertySourceFactory; +import lombok.SneakyThrows; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; + +/** + * TODO + * + * @Classname PacketCombineDslService + * @Date 2024/4/19 16:16 + * @Author wWei + */ +@Service +@PropertySource(value = "classpath:dsl-sql-template.sql", factory = SqlPropertySourceFactory.class) +public class PacketCombineDslServiceImpl implements PacketCombineDslService { + + private static final String DATAPATH_TELEMETRY_RECORD = "DATAPATH_TELEMETRY_RECORD"; + private static final String X_HOS_META_FILE_TYPE_KEY = "x-hos-meta-file-type"; + private static final String X_HOS_META_FILE_TYPE_VALUE_PCAPNG = "pcapng"; + private static final String TROUBLESHOOTING_FILE_BUCKET = "troubleshooting_file_bucket"; + private static final String FILED_MEASUREMENTS = "measurements"; + private static final String FILED_PACKET = "packet"; + private static final String FILED_TIMESTAMP_US = "timestamp_us"; + private static final String FILED_PACKET_LENGTH = "packet_length"; + private static final String SLED_IP = "sled_ip"; + private static final String DEVICE_GROUP = "device_group"; + private static final String TRAFFIC_LINK_ID = "traffic_link_id"; + + private static final Log log = LogFactory.get(); + private Environment environment; + private DatabaseService databaseService; + private SQLSyncQueryService sqlSyncQueryService; + private HosService hosService; + private EngineConfigSource engineConfigSource; + + @Override + public BaseResult run(DSLQueryRequestParam dslQueryRequestParam) { + if (engineConfigSource.getPacketCombineDelaySeconds() > 0) { + threadBlock(engineConfigSource.getPacketCombineDelaySeconds()); + } + String sql = buildSQL(dslQueryRequestParam); + List<Map<String, Object>> packets = getPackets(sql); + byte[] bytes = packetCombine(packets); + InputStream inputStream = new ByteArrayInputStream(bytes); + return uploadFile(dslQueryRequestParam.getId(), inputStream); + } + + @SneakyThrows + private void threadBlock(int delaySeconds) { + Thread.sleep(delaySeconds * 1000L); + } + + @Override + public List<Map<String, Object>> getPackets(String sql) { + BaseResult<List<Map<String, Object>>> baseResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder() + .originalSQL(sql) + .option(QueryOption.REAL_TIME.getValue()) + .build()); + if (baseResult.isSuccess()) { + return baseResult.getData(); + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "get packet records failed")); + } + + @Override + public byte[] packetCombine(List<Map<String, Object>> records) { + List<byte[]> packetList = new ArrayList<>(); + for (Map<String, Object> map : records) { + String measurements = JSONUtil.toJsonStr(map.get(FILED_MEASUREMENTS)); + Object sledIp = map.get(SLED_IP); + Object deviceGroup = map.get(DEVICE_GROUP); + Object trafficLinkId = map.get(TRAFFIC_LINK_ID); + JSONObject jsonObject = new JSONObject(); + jsonObject.set(SLED_IP, sledIp); + jsonObject.set(DEVICE_GROUP, deviceGroup); + jsonObject.set(TRAFFIC_LINK_ID, trafficLinkId); + int packetLen = Integer.parseInt(String.valueOf(map.get(FILED_PACKET_LENGTH))); + long timestampUS = Long.parseLong(String.valueOf(map.get(FILED_TIMESTAMP_US))); + String base64EncodedString = String.valueOf(map.get(FILED_PACKET)); + byte[] decodedBytes = Base64.getDecoder().decode(base64EncodedString); + byte[] bytes = generatePacketBlock(timestampUS, decodedBytes, packetLen, measurements, jsonObject); + packetList.add(bytes); + } + return mergeByte(packetList); + } + + @Override + public BaseResult uploadFile(String fileName, InputStream inputStream) { + Map<String, String> objectObjectHashMap = Maps.newHashMap(); + objectObjectHashMap.put(X_HOS_META_FILE_TYPE_KEY, X_HOS_META_FILE_TYPE_VALUE_PCAPNG); + BaseResult baseResult = hosService.uploadFile(TROUBLESHOOTING_FILE_BUCKET, fileName, inputStream, objectObjectHashMap); + if (!baseResult.isSuccess()) { + return baseResult; + } + return hosService.getFile(TROUBLESHOOTING_FILE_BUCKET, fileName); + } + + private String buildSQL(DSLQueryRequestParam dslQueryRequestParam) { + String dataSource = dslQueryRequestParam.getDataSource(); + DSLQueryContext dslQueryContext = BeanUtil.copyProperties(dslQueryRequestParam, DSLQueryContext.class); + return dslQueryContext.toSql( + environment.getProperty(DATAPATH_TELEMETRY_RECORD) + , dataSource + , databaseService.getPartitionKey(dataSource) + , "unix_timestamp"); + } + + private static byte[] mergeByte(List<byte[]> packetList) { + //拼接文件头信息和所有增强数据包块 + List<byte[]> pcapFileDataList = new ArrayList<>(); + //添加文件头信息 + pcapFileDataList.add(generatePCAPNGHeader()); + //添加所有增强数据包块 + pcapFileDataList.addAll(packetList); + //合并所有数据 + byte[][] newBytes = new byte[pcapFileDataList.size()][]; + pcapFileDataList.toArray(newBytes); + return ArrayUtil.addAll(newBytes); + } + + private static byte[] generatePacketBlock(long timestamp, byte[] blockData, int packetLen, String measurements, JSONObject jsonObject) { + //计算注释的长度 + List<byte[]> commentsList = new ArrayList<>(); + int commentsLength = 0; + //获取注释的json数组 + JSONArray measurementsJsonArray = JSONUtil.parseArray(measurements); + measurementsJsonArray.add(0, jsonObject); + for (Object measurement : measurementsJsonArray) { + byte[] comment = ((JSONObject) measurement).toJSONString(2).getBytes(); + if (comment.length > 65535) { + comment = "The comment exceeds the length limit of 65535 characters and cannot be displayed.".getBytes(); + } + //获取注释长度，补齐4字节 + int commentPadding = 0; + if (comment.length % 4 != 0) { + commentPadding = 4 - comment.length % 4; + } + int commentLength = comment.length + commentPadding; + commentsLength += commentLength + 4; + commentsList.add(comment); + } + //计算数据包块的填充字节数 + int blockDataLength = blockData.length; + int packetPadding = 0; + if (blockDataLength % 4 != 0) { + packetPadding = 4 - blockDataLength % 4; + } + //计算增强数据包块长度 + int packetBlockLength = commentsLength + blockDataLength + packetPadding + 32; + //生成增强数据包块 + ByteBuffer packetBlockBuffer = ByteBuffer.allocate(packetBlockLength); + packetBlockBuffer.order(ByteOrder.LITTLE_ENDIAN); + packetBlockBuffer.putInt(0x00000006);//增强数据包块块类型 + packetBlockBuffer.putInt(packetBlockLength);//增强数据包块长度 + packetBlockBuffer.putInt(0);//接口ID，从IDB来，默认为0 + packetBlockBuffer.putInt((int) (timestamp >> 32));//时间戳高32位 + packetBlockBuffer.putInt((int) (timestamp & 0xFFFFFFFFL));//时间戳低32位 + packetBlockBuffer.putInt(blockDataLength);//捕获的数据长度 + packetBlockBuffer.putInt(packetLen);//原始数据包长度 + packetBlockBuffer.put(blockData);//数据包 + //补齐4字节 + for (int i = 0; i < packetPadding; i++) { + packetBlockBuffer.put((byte) 0); + } + //注释数据 + for (byte[] comment : commentsList) { + packetBlockBuffer.putChar((char) 1);//选项类型，注释固定为1 + packetBlockBuffer.putChar((char) comment.length);//注释长度 + packetBlockBuffer.put(comment);//注释内容 + //计算注释的填充字节数 + int commentPadding = 0; + if (comment.length % 4 != 0) { + commentPadding = 4 - comment.length % 4; + } + //补齐4字节 + for (int i = 0; i < commentPadding; i++) { + packetBlockBuffer.put((byte) 0); + } + } + packetBlockBuffer.putInt(packetBlockLength);//增强数据包块长度 + return packetBlockBuffer.array(); + } + + private static byte[] generatePCAPNGHeader() { + ByteBuffer buffer = ByteBuffer.allocate(48); + buffer.order(ByteOrder.LITTLE_ENDIAN); + //节标题块标识 + buffer.putInt(0x0A0D0D0A); + //节标题块长度 + buffer.putInt(28); + //块的数据顺序，大端或者小端模式,0x4D3C2B1A为小端，0x1A2B3C4D为大端 + buffer.putInt(0x1A2B3C4D); + //主版本信息 + buffer.putShort((short) 1); + //次版本信息 + buffer.putShort((short) 0); + //下一块长度，-1表示未指定 + buffer.putLong(-1); + //节标题块长度 + buffer.putInt(28); + //接口描述块标识 + buffer.putInt(1); + //接口描述块长度 + buffer.putInt(20); + //链路类型 + buffer.putInt(1); + //从每个数据包转储的最大字节数。每个数据包超过此值的部分将不会存储在文件中。 + buffer.putInt(0); + //接口描述块长度 + buffer.putInt(20); + return buffer.array(); + } + + @Autowired + public void setEnvironment(Environment environment) { + this.environment = environment; + } + + @Autowired + public void setDatabaseService(DatabaseService databaseService) { + this.databaseService = databaseService; + } + + @Autowired + public void setSqlSyncQueryService(SQLSyncQueryService sqlSyncQueryService) { + this.sqlSyncQueryService = sqlSyncQueryService; + } + + @Autowired + public void setHosService(HosService hosService) { + this.hosService = hosService; + } + + @Autowired + public void setEngineConfigSource(EngineConfigSource engineConfigSource) { + this.engineConfigSource = engineConfigSource; + } +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/QueryJobServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/QueryJobServiceImpl.java new file mode 100644 index 00000000..93c5fe80 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/QueryJobServiceImpl.java @@ -0,0 +1,494 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.util.BooleanUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.StringUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.jfinal.plugin.activerecord.Db; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.enums.*; +import com.mesalab.common.utils.HazelcastInstanceMapUtil; +import com.mesalab.common.utils.RandomUtil; +import com.mesalab.common.utils.sqlparser.AutoPeriodHelper; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.*; +import com.mesalab.qgw.service.*; +import com.mesalab.services.common.enums.MetricFunction; +import com.mesalab.services.common.enums.MetricType; +import com.mesalab.services.common.property.SqlPropertySourceFactory; +import com.mesalab.services.configuration.JobConfig; +import com.mesalab.services.service.JobService; +import com.mesalab.services.service.impl.JobExecuteService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.EnvironmentAware; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.env.Environment; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +/** + * TODO + * + * @Classname QueryJobServiceImpl + * @Date 2023/12/6 18:10 + * @Author wWei + */ +@Service +@PropertySource(value = "classpath:job-sql-template.sql", factory = SqlPropertySourceFactory.class) +public class QueryJobServiceImpl implements QueryJobService, EnvironmentAware { + private static final Log log = LogFactory.get(); + private Environment env; + private SQLSyncQueryService sqlSyncQueryService; + private JobService jobService; + private JobExecuteService jobExecuteService; + private DSLService dslService; + private TrafficSpectrumDslService trafficSpectrumDslService; + private EngineConfigSource engineConfigSource; + private JobConfig jobConfig; + + + @Override + public BaseResult createSQLSavedQuery(SqlQueryRequestParam request) { + initAndValidationID(request); + syntaxParseSql(request.getStatement()); + String sql = JSONUtil.quote(request.getStatement(), false).replace("'", "\\'"); + long currentTime = System.currentTimeMillis() / 1000; + Db.update(String.format(Objects.requireNonNull(env.getProperty("SAVED_QUERY_JOB_INIT")), request.getId(), sql, currentTime, currentTime)); + log.info("Committed a SQL Saved Query Job, id is: {}", request.getId()); + return BaseResultGenerator.successCreate(buildJobInfoOfCreated(request.getId())); + } + + @Override + public BaseResult createSQLAdHocQuery(SqlQueryRequestParam sqlQueryRequestParam) { + initAndValidationID(sqlQueryRequestParam); + String option = sqlQueryRequestParam.isDryRun() + ? QueryOption.SYNTAX_VALIDATION.getValue() : QueryOption.REAL_TIME.getValue(); + long start = System.currentTimeMillis(); + ExecutionMode execMode = sqlQueryRequestParam.getExecutionMode(); + if (ExecutionMode.ONESHOT.equals(execMode)) { + return sqlSyncQueryService.executeQuery(SQLQueryContext.builder() + .originalSQL(sqlQueryRequestParam.getStatement()) + .option(option) + .format(sqlQueryRequestParam.getOutputMode().getValue()) + .build()); + } else if (ExecutionMode.NORMAL.equals(execMode)) { + HazelcastInstanceMapUtil.put(sqlQueryRequestParam.getId(), new QueryCache(sqlQueryRequestParam.getId())); + jobExecuteService.addExecutorSql(sqlQueryRequestParam); + return BaseResultGenerator.successCreate(buildJobInfoOfCreated(sqlQueryRequestParam.getId())); + } else if (ExecutionMode.BLOCKING.equals(execMode)) { + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(sqlQueryRequestParam.getStatement()).option(option).format(sqlQueryRequestParam.getOutputMode().getValue()).build()); + QueryCache queryCache = buildQueryCacheOfDone(sqlQueryRequestParam.getId(), start, baseResult); + HazelcastInstanceMapUtil.put(sqlQueryRequestParam.getId(), queryCache); + return BaseResultGenerator.successCreate(buildJobInfoOfCreated(sqlQueryRequestParam.getId())); + } + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getMessage(), "not Supported")); + } + + @Override + public BaseResult createDSLAdHocQuery(DSLQueryRequestParam request) { + initAndValidationID(request); + AutoPeriodHelper.buildDslGranularity(request); + long start = System.currentTimeMillis(); + ExecutionMode execMode = request.getExecutionMode(); + if (ExecutionMode.ONESHOT.equals(execMode)) { + if (JobConfig.FIELD_DISCOVERY.equals(request.getName())) { + execAdnWaitFieldDiscoveryDone(request.getId(), request); + QueryCache queryCache = rebuildFieldDiscoveryQueryCache(getAdhocQueryCacheAndRefresh(request.getId())); + HazelcastInstanceMapUtil.remove(request.getId()); + BaseResult<Object> baseResult = queryCache.getBaseResult(); + Map<String, Object> job = baseResult.getJob(); + if (!BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_CANCELED))) + && !BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_FAILED))) + && BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_DONE)))) { + return BaseResultGenerator.success(baseResult.getStatistics(), null, baseResult.getOutputMode(), baseResult.getMeta(), baseResult.getData()); + } + return BaseResultGenerator.error(baseResult.getMessage()); + } else if (JobConfig.DATAPATH_PACKET_COMBINE.equals(request.getName())) { + return jobExecuteService.addDslExecutorPacketCombineWithoutCache(request); + } else if (JobConfig.TRAFFIC_SPECTRUM_SUMMARY.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_UNIQUE_CLIENT_AND_SERVER_IPS.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_APP_DISTRIBUTION.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE.equals(request.getName())) { + QueryCache queryCache = new QueryCache(request.getId()); + queryCache.setType(request.getName()); + HazelcastInstanceMapUtil.put(request.getId(), queryCache); + CountDownLatch countDownLatch = new CountDownLatch(1); + jobExecuteService.addDslExecutorTrafficSpectrumWithCache(request, countDownLatch); + try { + boolean await = countDownLatch.await(jobConfig.getExecutionTimeout(), TimeUnit.MILLISECONDS); + BaseResult baseResult = getAdHocQueryResultById(request.getId()); + HazelcastInstanceMapUtil.remove(request.getId()); + if (await) { + Map<String, Object> job = baseResult.getJob(); + if (!BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_CANCELED))) + && !BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_FAILED))) + && BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_DONE)))) { + return BaseResultGenerator.success(baseResult.getStatistics(), null, baseResult.getOutputMode(), baseResult.getMeta(), baseResult.getData()); + } + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Job was timeout.")); + + } catch (InterruptedException e) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Job was Interrupted. message: " + e.getMessage())); + } + } else { + DSLQueryContext dslQueryContext = BeanUtil.copyProperties(request, DSLQueryContext.class); + return dslService.execDsl(dslQueryContext, request.isDryRun()); + } + } else if (ExecutionMode.NORMAL.equals(execMode)) { + QueryCache queryCache = new QueryCache(request.getId()); + if (JobConfig.FIELD_DISCOVERY.equals(request.getName())) { + queryCache.setType(JobConfig.FIELD_DISCOVERY); + validFieldDiscovery(request); + HazelcastInstanceMapUtil.put(request.getId(), queryCache); + jobExecuteService.addExecutorFieldDiscovery(request); + } else if (JobConfig.DATAPATH_PACKET_COMBINE.equals(request.getName())) { + HazelcastInstanceMapUtil.put(request.getId(), queryCache); + jobExecuteService.addExecutorDslPacketCombineWithCache(request, null); + } else if (JobConfig.TRAFFIC_SPECTRUM_SUMMARY.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_UNIQUE_CLIENT_AND_SERVER_IPS.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_APP_DISTRIBUTION.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE.equals(request.getName())) { + queryCache.setType(request.getName()); + HazelcastInstanceMapUtil.put(request.getId(), queryCache); + jobExecuteService.addDslExecutorTrafficSpectrumWithCache(request, null); + } else { + HazelcastInstanceMapUtil.put(request.getId(), queryCache); + jobExecuteService.addExecutorDsl(request); + } + return BaseResultGenerator.successCreate(buildJobInfoOfCreated(request.getId())); + } else if (ExecutionMode.BLOCKING.equals(execMode)) { + if (JobConfig.FIELD_DISCOVERY.equals(request.getName())) { + execAdnWaitFieldDiscoveryDone(request.getId(), request); + return BaseResultGenerator.successCreate(buildJobInfoOfCreated(request.getId())); + } else if (JobConfig.DATAPATH_PACKET_COMBINE.equals(request.getName())) { + QueryCache queryCache = new QueryCache(request.getId()); + HazelcastInstanceMapUtil.put(request.getId(), queryCache); + CountDownLatch countDownLatch = new CountDownLatch(1); + jobExecuteService.addExecutorDslPacketCombineWithCache(request, countDownLatch); + try { + boolean await = countDownLatch.await(jobConfig.getExecutionTimeout(), TimeUnit.MILLISECONDS); + if (await) { + return BaseResultGenerator.successCreate(buildJobInfoOfCreated(request.getId())); + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Job was timeout.")); + } catch (InterruptedException e) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Job was Interrupted. message: " + e.getMessage())); + } + } else if (JobConfig.TRAFFIC_SPECTRUM_SUMMARY.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_UNIQUE_CLIENT_AND_SERVER_IPS.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_APP_DISTRIBUTION.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND.equals(request.getName()) + || JobConfig.TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE.equals(request.getName())) { + QueryCache queryCache = new QueryCache(request.getId()); + queryCache.setType(request.getName()); + HazelcastInstanceMapUtil.put(request.getId(), queryCache); + CountDownLatch countDownLatch = new CountDownLatch(1); + jobExecuteService.addDslExecutorTrafficSpectrumWithCache(request, countDownLatch); + try { + boolean await = countDownLatch.await(jobConfig.getExecutionTimeout(), TimeUnit.MILLISECONDS); + if (await) { + return BaseResultGenerator.successCreate(buildJobInfoOfCreated(request.getId())); + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Job was timeout.")); + } catch (InterruptedException e) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Job was Interrupted. message: " + e.getMessage())); + } + } else { + DSLQueryContext dslQueryContext = BeanUtil.copyProperties(request, DSLQueryContext.class); + BaseResult baseResult = dslService.execDsl(dslQueryContext, request.isDryRun()); + QueryCache queryCache = buildQueryCacheOfDone(request.getId(), start, baseResult); + HazelcastInstanceMapUtil.put(request.getId(), queryCache); + return BaseResultGenerator.successCreate(buildJobInfoOfCreated(request.getId())); + } + } + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getMessage(), "not Supported")); + } + + private void execAdnWaitFieldDiscoveryDone(String id, DSLQueryRequestParam request) { + QueryCache queryCacheStart = new QueryCache(id); + queryCacheStart.setType(JobConfig.FIELD_DISCOVERY); + validFieldDiscovery(request); + HazelcastInstanceMapUtil.put(id, queryCacheStart); + Future<Boolean> booleanFuture = jobExecuteService.addExecutorFieldDiscovery(request); + while (true) { + if (!booleanFuture.isDone()) { + continue; + } + break; + } + } + + private QueryCache buildQueryCacheOfDone(String id, long start, BaseResult baseResult) { + QueryCache queryCache = new QueryCache(id); + Map<String, Object> jobInfo = queryCache.getBaseResult().getJob(); + jobInfo.put(JobConfig.IS_DONE, true); + jobInfo.put(JobConfig.DONE_PROGRESS, 1); + if (!baseResult.isSuccess()) { + jobInfo.put(JobConfig.IS_FAILED, false); + jobInfo.put(JobConfig.REASON, baseResult.getMessage()); + } + jobInfo.put(JobConfig.START_TIME, DateUtils.convertTimestampToString(start / 1000, DatePattern.UTC_PATTERN)); + jobInfo.put(JobConfig.END_TIME, DateUtils.convertTimestampToString(System.currentTimeMillis() / 1000, DatePattern.UTC_PATTERN)); + baseResult.setJob(jobInfo); + queryCache.setBaseResult(baseResult); + return queryCache; + } + + @Override + public BaseResult getAdHocQueryResultById(String id) { + QueryCache queryCache = getAdhocQueryCacheAndRefresh(id); + if (queryCache == null) { + return BaseResultGenerator.success(Lists.newArrayList()); + } + if (JobConfig.FIELD_DISCOVERY.equals(queryCache.getType())) { + queryCache = rebuildFieldDiscoveryQueryCache(queryCache); + } else if (JobConfig.TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE.equals(queryCache.getType())) { + queryCache = rebuildTrafficSpectrumCIPConnectAppUsageQueryCache(queryCache); + } + BaseResult<Object> baseResult = queryCache.getBaseResult(); + return BaseResultGenerator.success(baseResult.getStatistics(), baseResult.getJob(), baseResult.getOutputMode(), baseResult.getMeta(), baseResult.getData()); + } + + @Override + public BaseResult getSavedQueryResultById(String id) { + return jobService.getSavedQueryResult(id); + } + + @Override + public BaseResult getAdHocQueryStatusById(String id) { + QueryCache queryCache = getAdhocQueryCacheAndRefresh(id); + if (queryCache == null) { + return BaseResultGenerator.success(Lists.newArrayList()); + } + return BaseResultGenerator.success(null, queryCache.getBaseResult().getJob(), null, null, null); + } + + @Override + public BaseResult getSavedQueryStatusById(String id) { + return jobService.getSavedQueryStatus(id); + } + + @Override + public BaseResult getSavedQueryResult(List<String> ids) { + List<Map<String, Object>> result = jobService.batchSavedQueryResult(ids); + return BaseResultGenerator.success(result); + } + + @Override + public BaseResult getAdHocQueryResult(List<String> ids) { + List<Object> results = Lists.newArrayList(); + for (String id : ids) { + BaseResult<Object> baseResult; + QueryCache queryCache = getAdhocQueryCacheAndRefresh(id); + if (queryCache == null) { + continue; + } + if (JobConfig.FIELD_DISCOVERY.equals(queryCache.getType())) { + queryCache = rebuildFieldDiscoveryQueryCache(queryCache); + } + baseResult = queryCache.getBaseResult(); + Map<String, Object> resultItem = Maps.newLinkedHashMap(); + resultItem.put(JobConfig.STATISTICS, baseResult.getStatistics()); + resultItem.put(JobConfig.JOB, baseResult.getJob()); + resultItem.put(JobConfig.META, baseResult.getMeta()); + resultItem.put(JobConfig.DATA, baseResult.getData()); + results.add(resultItem); + } + return BaseResultGenerator.success(results); + } + + @Override + public BaseResult getSavedQueryStatus(List<String> ids) { + List<Map<String, Object>> result = jobService.batchSavedQueryStatus(ids); + return BaseResultGenerator.success(result); + } + + @Override + public BaseResult getAdHocQueryStatus(List<String> ids) { + List<Object> results = Lists.newArrayList(); + for (String id : ids) { + QueryCache queryCache = getAdhocQueryCacheAndRefresh(id); + if (queryCache == null) { + continue; + } + results.add(queryCache.getBaseResult().getJob()); + } + return BaseResultGenerator.success(results); + } + + @Override + public BaseResult deleteSavedQueryById(String id) { + return jobService.cancelSavedQuery(id); + } + + private void validFieldDiscovery(DSLQueryRequestParam request) { + Object filter = request.getFilter(); + if (StrUtil.isEmptyIfStr(filter)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_CUSTOM_FIELD_DISCOVERY_ERROR)); + } + + Object metric = request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC); + if (StringUtil.isEmpty(metric)) { + return; + } + boolean validMetric = MetricType.isValid(String.valueOf(metric)); + boolean validFn = MetricFunction.isValid(String.valueOf(request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC_FN))); + if (!validMetric || !validFn) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_CUSTOM_FIELD_DISCOVERY_ERROR)); + } + Object fields = request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS); + if (StrUtil.isEmptyIfStr(fields)) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_CUSTOM_FIELD_DISCOVERY_ERROR)); + } + List<String> fieldList = (List<String>) request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS); + request.getCustomRequestParam().put(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS, fieldList.stream().distinct().collect(Collectors.toList())); + } + + private void syntaxParseSql(String sql) { + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(sql).option(QueryOption.SYNTAX_PARSE.getValue()).build()); + if (!baseResult.isSuccess()) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), baseResult.getMessage())); + } + } + + private static String generateJobId() { + return RandomUtil.getUUID(); + } + + private static Map<String, Object> buildJobInfoOfCreated(String id) { + Map<String, Object> result = Maps.newHashMap(); + result.put(JobConfig.JOB_ID, id); + return result; + } + + private static QueryCache getAdhocQueryCacheAndRefresh(String id) { + try { + HazelcastInstanceMapUtil.retrieveMap().lock(id); + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + if (queryCache == null) { + return null; + } + queryCache.setLatestQueryTimeMs(System.currentTimeMillis()); + HazelcastInstanceMapUtil.put(id, queryCache); + return queryCache; + } finally { + HazelcastInstanceMapUtil.retrieveMap().unlock(id); + } + } + + private static QueryCache rebuildFieldDiscoveryQueryCache(QueryCache queryCache) { + BaseResult<Object> baseResult = queryCache.getBaseResult(); + Object dataObj = baseResult.getData(); + if (!StrUtil.isEmptyIfStr(dataObj)) { + List<Map> dataList = (List<Map>) dataObj; + for (Map map : dataList) { + for (Object itemObj : map.keySet()) { + Map<String, Object> itemMap = (Map) map.get(itemObj); + Object topkObj = itemMap.get("topk"); + if (!StrUtil.isEmptyIfStr(topkObj)) { + List<Map<String, Object>> topkList = (List<Map<String, Object>>) topkObj; + List<Map<String, Object>> list = topkList.subList(0, Math.min(10, topkList.size())); + itemMap.put("topk", Lists.newArrayList(list)); + } + } + } + return queryCache; + } + return queryCache; + } + + private QueryCache rebuildTrafficSpectrumCIPConnectAppUsageQueryCache(QueryCache queryCache) { + BaseResult<Object> baseResult = queryCache.getBaseResult(); + Object dataObj = baseResult.getData(); + if (StringUtil.isNotEmpty(baseResult.getData())) { + List<Map> dataList = (List<Map>) dataObj; + Map internalExternalBipartiteGraph = trafficSpectrumDslService.getInternalExternalBipartiteGraph(dataList.get(0), engineConfigSource.getTrafficSpectrumClientIPAppResultNum()); + queryCache.getBaseResult().setData(Lists.newArrayList(internalExternalBipartiteGraph)); + return queryCache; + } + return queryCache; + } + + private static void initAndValidationID(CommonRequestParam request) { + String id = StrUtil.isBlank(request.getId()) ? generateJobId() : request.getId(); + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + if (queryCache != null) { + Map<String, Object> job = queryCache.getBaseResult().getJob(); + boolean isFailed = BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_FAILED))); + if (!isFailed) { + throw new QGWBusinessException(HttpStatusCodeEnum.REQ_FORBIDDEN.getCode(), CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_PARAM_SYNTAX_EXCEPTION.getMessage(), "Job id already exists")); + } + } + request.setId(id); + } + + @Override + public void setEnvironment(Environment environment) { + this.env = environment; + } + + @Autowired + public void setJobService(JobService jobService) { + this.jobService = jobService; + } + + @Autowired + public void setJobExecuteService(JobExecuteService jobExecuteService) { + this.jobExecuteService = jobExecuteService; + } + + @Autowired + public void setSqlSyncQueryService(SQLSyncQueryService sqlSyncQueryService) { + this.sqlSyncQueryService = sqlSyncQueryService; + } + + @Autowired + public void setDslService(DSLService dslService) { + this.dslService = dslService; + } + + @Autowired + public void setTrafficSpectrumDslService(TrafficSpectrumDslService trafficSpectrumDslService) { + this.trafficSpectrumDslService = trafficSpectrumDslService; + } + + @Autowired + public void setEngineConfigSource(EngineConfigSource engineConfigSource) { + this.engineConfigSource = engineConfigSource; + } + + @Autowired + public void setJobConfig(JobConfig jobConfig) { + this.jobConfig = jobConfig; + } +}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/qgw/service/impl/QueryServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/QueryServiceImpl.java deleted file mode 100644 index 6c11cac9..00000000 --- a/src/main/java/com/mesalab/qgw/service/impl/QueryServiceImpl.java +++ /dev/null @@ -1,618 +0,0 @@ -package com.mesalab.qgw.service.impl; - -import cn.hutool.core.exceptions.ExceptionUtil; -import cn.hutool.core.text.StrFormatter; -import cn.hutool.core.util.BooleanUtil; -import cn.hutool.core.util.StrUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.google.common.base.CaseFormat; -import com.google.common.base.Joiner; -import com.google.common.collect.Maps; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.entity.SchemaBase; -import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.common.enums.*; -import com.mesalab.common.utils.sqlparser.SQLFunctionUtil; -import com.mesalab.common.utils.sqlparser.SQLHelper; -import com.mesalab.common.utils.sqlparser.SQLSyntaxParserUtil; -import com.mesalab.qgw.dialect.Dialect; -import com.mesalab.qgw.dialect.FederationDialect; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.qgw.model.basic.*; -import com.mesalab.qgw.model.basic.udf.UDF; -import com.mesalab.qgw.model.basic.udf.UDFElements; -import com.mesalab.qgw.service.QueryService; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.qgw.service.SystemService; -import com.geedgenetworks.utils.StringUtil; -import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.expression.BinaryExpression; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.Function; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.DescribeStatement; -import net.sf.jsqlparser.statement.ExplainStatement; -import net.sf.jsqlparser.statement.ShowStatement; -import net.sf.jsqlparser.statement.Statement; -import net.sf.jsqlparser.statement.select.*; -import net.sf.jsqlparser.util.TablesNamesFinder; -import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.lang.reflect.Constructor; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -@Service("queryService") -public class QueryServiceImpl implements QueryService { - private static final Log log = LogFactory.get(); - @Autowired - private MetadataService metadataServiceTemp; - @Autowired - private SystemService systemServiceTemp; - - private static MetadataService metadataService; - private static SystemService systemService; - - @PostConstruct - private void initMetadataService() { - metadataService = this.metadataServiceTemp; - systemService = this.systemServiceTemp; - } - - public static final int MAX_PARSER_LEVEL = 5; - public static final String PARSER_EXCEPTION_CLASS_NAME = "net.sf.jsqlparser.parser.ParseException:"; - public static final String PARSER_EXCEPTION_IDENTIFICATION = "Was expecting one of:"; - public static final Pattern PATTERN_ANGLE_BRACKET = Pattern.compile("<(.*?)>", Pattern.CASE_INSENSITIVE); - - - @Override - public BaseResult executeQuery(QueryProfile queryProfile) { - validateAndSetDefaultValue(queryProfile); - init(queryProfile); - Statement statement; - try { - statement = CCJSqlParserUtil.parse(queryProfile.getQuery()); - } catch (JSQLParserException e) { - log.error("SQL Syntax Error: {}", e); - String message = getJSQLParserSimpleErrorMessage(e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),message)); - } - BaseResult baseResult; - log.debug("Request option ：{}, SQL ：{}", queryProfile.getOption(), queryProfile.getQuery()); - if (statement instanceof Select) { - baseResult = executeSelectStatement(queryProfile); - } else if (statement instanceof DescribeStatement) { - DescribeStatement describeStatement = (DescribeStatement) statement; - baseResult = executeDescStatement(queryProfile, describeStatement); - } else if (statement instanceof ShowStatement) { - baseResult = executeShowStatement(queryProfile); - } else if (statement instanceof ExplainStatement) { - baseResult = executeExplainStatement(queryProfile); - } else { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),QGWMessageConst.ONLY_SUPPORT_STATEMENT_OPERATIONS_FOR_SELECT)); - } - return baseResult; - } - - private void init(QueryProfile param) { - if(SQLHelper.getTableName(param.getQuery()).isEmpty()){ - return; - } - String tableName = SQLHelper.getTableName(param.getQuery()).get(0); - param.setDbType(metadataService.getDBTypeByTableName(tableName)); - param.setDialectDBType(SQLHelper.getDialectDBType(param.getDbType())); - } - - private void validateAndSetDefaultValue(QueryProfile param) { - if (StringUtil.isBlank(param.getOption())) { - param.setOption(QueryOptionEnum.REAL_TIME.getValue()); - } - if (StringUtil.isBlank(param.getFormat())) { - param.setFormat(QueryFormatEnum.JSON.getValue()); - } - if (StringUtil.isBlank(param.getQuery())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.SQL_MUST_BE_EXISTS)); - } - if (Arrays.stream(QueryOptionEnum.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(param.getOption()))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.OPTION_TYPE_ERROR)); - } - if (Arrays.stream(QueryFormatEnum.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(param.getFormat()))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.FORMAT_TYPE_ERROR)); - } - } - - - /** - * Desc: 简化JSQLParser语法异常信息 - * - * @param e - * @return {@link String} - * @created by wWei - * @date 2021/4/26 9:49 上午 - */ - private String getJSQLParserSimpleErrorMessage(Throwable e) { - String message = Joiner.on(" ").skipNulls().join(ExceptionUtil.getSimpleMessage(e), ExceptionUtil.getRootCauseMessage(e)); - message = message.replace(PARSER_EXCEPTION_CLASS_NAME, StringUtil.EMPTY); - int end = message.indexOf(PARSER_EXCEPTION_IDENTIFICATION); - message = message.substring(0, end < 0 ? message.length() : end); - message = StrUtil.removeAllLineBreaks(message); - Matcher matcher = PATTERN_ANGLE_BRACKET.matcher(message); - if (matcher.find()) { - message = matcher.replaceAll(StringUtil.EMPTY); - } - String[] split = message.split(" "); - StringBuilder stringBuilder = new StringBuilder(message.length()); - for (String c : split) { - if (StringUtil.isBlank(c)) { - continue; - } - stringBuilder.append(" ").append(c); - } - return stringBuilder.toString().trim(); - } - - /** - * 1.设置默认queryID - * 2.为Select操作选择不同的执行动作，包含SQL解析(Syntax-parse)、SQL验证（Syntax-validation）、实时查询(Real-time)、长查询(Long-term) - * @param param 请求参数 - * @return BaseResult 执行结果 - */ - private BaseResult executeSelectStatement(QueryProfile param) { - if (QueryOptionEnum.SYNTAX_PARSE.getValue().equalsIgnoreCase(param.getOption())) { - return executeSyntaxParse(param); - } else if (QueryOptionEnum.SYNTAX_VALIDATION.getValue().equalsIgnoreCase(param.getOption())) { - return executeSyntaxValidation(param); - } else if (QueryOptionEnum.LONG_TERM.getValue().equalsIgnoreCase(param.getOption())) { - param.setQueryId(systemService.getCustomQueryId(param.getResultId(), param.getQuery())); - return executeLongTermQuery(param); - } else { - return executeRealTimeQuery(param); - } - - } - - private BaseResult executeSyntaxParse(QueryProfile param) { - return BaseResultGenerator.success(SQLSyntaxParserUtil.syntaxParse(param.getQuery())); - } - - private BaseResult executeSyntaxValidation(QueryProfile param) { - return new FederationDialect(param, getDBDialect(param).executeSyntaxCheck()).executeSyntaxCheck(); - } - - private BaseResult executeLongTermQuery(QueryProfile param) { - return new FederationDialect(param, getDBDialect(param).executeQuery()).executeQuery(); - } - - - private BaseResult executeRealTimeQuery(QueryProfile param) { - if (BooleanUtil.isTrue(param.getSampled())) { - return new FederationDialect(param, getDBDialect(param).executeSampleQuery()).executeSampleQuery(); - } else { - return new FederationDialect(param, getDBDialect(param).executeQuery()).executeQuery(); - } - } - - - private Dialect getDBDialect(QueryProfile param) { - convertQueryRecursive(param, parserSQLByAst(param.getQuery())); - return getReflectiveDialectObject(param); - } - - - private Dialect getReflectiveDialectObject(QueryProfile param) { - Dialect dialect; - try { - Class dialectClazz = Class.forName("com.mesalab.qgw.dialect." + - CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, param.getDbType()) + "Dialect"); - Constructor constructor = dialectClazz.getConstructor(QueryProfile.class); - dialect = (Dialect) constructor.newInstance(param); - } catch (ReflectiveOperationException | RuntimeException e) { - log.error("Dialect conversion instance exception:{}",e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),e.getMessage())); - } - return dialect; - } - - - - - - /** - * 递归解析SQLQuerySource ，拆分为数据库执行SQL与引擎执行的SQL source - * - * @param sqlQuerySource SQL解析对象 - */ - public static void convertQueryRecursive(QueryProfile param, SQLQuerySource sqlQuerySource) { - SQLQuerySource engineQuerySource = null; - SQLQuerySource dbQuerySource = null; - - SQLQuerySource indexUpSqlQuerySource = null; - SQLQuerySource indexSqlQuerySource = sqlQuerySource; - for (int i = 0; i < MAX_PARSER_LEVEL; i++) { - if (indexSqlQuerySource.getUdfSet().size() > 0) { - dbQuerySource = indexSqlQuerySource; - break; - } - if (StringUtil.isEmpty(indexSqlQuerySource.getSubSelect())) { - dbQuerySource = sqlQuerySource; - indexUpSqlQuerySource = null; - break; - } else { - indexUpSqlQuerySource = indexSqlQuerySource; - indexSqlQuerySource = indexSqlQuerySource.getSubSqlQuerySources().get(0); - if (indexSqlQuerySource.getUdfSet().size() > 0) { - dbQuerySource = indexSqlQuerySource; - break; - } - } - } - - if (StringUtil.isNotEmpty(indexUpSqlQuerySource)) { - engineQuerySource = new SQLQuerySource(); - String tableName = StringUtil.isEmpty(indexUpSqlQuerySource.getSubSelect().getAlias()) ? indexUpSqlQuerySource.getTableNames().get(0) : indexUpSqlQuerySource.getSubSelect().getAlias().getName(); - String replace = sqlQuerySource.getSqlBody().replace(indexUpSqlQuerySource.getSubSelect().toString(), tableName); - engineQuerySource.setSqlBody(replace); - engineQuerySource.getTableNames().add(0,tableName); - engineQuerySource.setLimit(sqlQuerySource.getLimit()); - } - - param.setDbQuerySource(dbQuerySource); - param.setEngineQuerySource(engineQuerySource); - } - - - /** - * 解析SQL - * - * @param sql - * @return - */ - public static SQLQuerySource parserSQLByAst(String sql) { - - - SQLQuerySource sqlQuerySource = new SQLQuerySource(); - try { - - Statement statement = CCJSqlParserUtil.parse(sql); - Select selectStatement = null; - if (statement instanceof Select) { - selectStatement = (Select) statement; - } else if (statement instanceof ExplainStatement) { - selectStatement = ((ExplainStatement) statement).getStatement(); - } else { - log.error("Not support DML Parser"); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), QGWMessageConst.SQL_PARSE_ONLY_SUPPORT_SELECT)); - } - - sqlQuerySource.setSqlBody(String.valueOf(selectStatement.getSelectBody())); - TablesNamesFinder tablesNamesFinder = new TablesNamesFinder(); - List<String> tableList = tablesNamesFinder.getTableList(selectStatement); - tableList.forEach(o -> { - int i = o.lastIndexOf("."); - sqlQuerySource.getTableNames().add(i < 0 ? o : o.substring(i + 1)); - }); - sqlQuerySource.setPartitionKey(metadataService.getPartitionKey(sqlQuerySource.getTableNames().get(0))); - SelectBody body = selectStatement.getSelectBody(); - if (body instanceof PlainSelect) { //单条查询 - - PlainSelect select = (PlainSelect) body; - - FromItem fromItem = select.getFromItem(); - sqlQuerySource.setFromItem(fromItem); - sqlQuerySource.setJoins(select.getJoins()); - Expression where = select.getWhere(); - - if (where != null) { - sqlQuerySource.setWhereExpression(where); - } - - List<SelectItem> selectItemsList = select.getSelectItems(); - sqlQuerySource.setSelectItems(selectItemsList); - - for (SelectItem item : selectItemsList) { - if (item instanceof SelectExpressionItem) { - SelectExpressionItem expressionItem = ((SelectExpressionItem) item); - if (StringUtil.isNotEmpty(expressionItem.getAlias())) { - String aliasName = expressionItem.getAlias().getName(); - if ((aliasName.startsWith("\"") && aliasName.endsWith("\"")) - || ((aliasName.startsWith("`") && aliasName.endsWith("`")))) { - aliasName = aliasName.substring(1, aliasName.length() - 1); - } - sqlQuerySource.getAliasFields().put(aliasName, expressionItem.getExpression().toString()); - } else { - sqlQuerySource.getAliasFields().put(expressionItem.getExpression().toString(), expressionItem.getExpression().toString()); - } - addUDFSet(sqlQuerySource.getUdfSet(), expressionItem.getExpression());//伪代码：后期需要在SQLQuerySource where对象中提取 - - } - } - - GroupByElement groupBy; - if (StringUtil.isNotEmpty(groupBy = select.getGroupBy())) { - sqlQuerySource.setGroupByElement(groupBy); - } - sqlQuerySource.setOrderByElements(select.getOrderByElements()); - FromItem subItem = select.getFromItem(); - - if (subItem instanceof SubSelect) { - SubSelect subSelect = (SubSelect) subItem; - log.debug("存在子查询为: {}", subSelect); - sqlQuerySource.setSubSelect(subSelect); - sqlQuerySource.getSubSqlQuerySources().add(0, parserSQLByAst(String.valueOf(subSelect.getSelectBody()))); - } - - - Limit limit = select.getLimit(); - - - if (limit != null) { - - if (StringUtil.isNotEmpty(limit.getOffset())) { - sqlQuerySource.setLimit(StringUtil.setDefaultIfEmpty(limit.getOffset(), 0) + "," + limit.getRowCount()); - } else { - sqlQuerySource.setLimit(String.valueOf(limit.getRowCount())); - } - } - - - } else if (body instanceof SetOperationList) { // 连接查询 - SetOperationList setOperationList = (SetOperationList) body; - List<SelectBody> selects = setOperationList.getSelects(); - //暂时只解析第一个结构，不接受不相同的where - if (StringUtil.isNotEmpty(selects)) { - SQLQuerySource parseSql = parserSQLByAst(selects.get(0).toString()); - sqlQuerySource.setUdfSet(parseSql.getUdfSet()); - sqlQuerySource.setWhereExpression(parseSql.getWhereExpression()); - sqlQuerySource.setGroupByElement(parseSql.getGroupByElement()); - sqlQuerySource.setAliasFields(parseSql.getAliasFields()); - } - sqlQuerySource.setLimit(SQLHelper.INVALID_LIMIT_DESC); - } else { //其它暂不支持 - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), QGWMessageConst.SQL_PARSE_ONLY_SUPPORT_SELECT)); - } - - - } catch (JSQLParserException | RuntimeException e) { - log.error("sqlParser error: {}", e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),e.getMessage())); - - } - - return sqlQuerySource; - } - - - /** - * 自定义函数封装 - * - * @param udfSet - * @param expr - */ - private static void addUDFSet(Set<UDF> udfSet, Expression expr) { - parserExpressionForFun(udfSet, expr); - } - - private static void parserExpressionForFun(Set<UDF> udfSet, Expression expr) { - if (expr instanceof Function) { - Function fun = (Function) expr; - if (SQLFunctionUtil.functions.keySet().contains(fun.getName().toUpperCase())) { - List<Expression> expressions = fun.getParameters().getExpressions(); - UDF udf = getUDF(fun, expressions); - udfSet.add(udf); - } - if (fun.getParameters() == null) { - return; - } - for (Expression expression : fun.getParameters().getExpressions()) { - parserExpressionForFun(udfSet, expression); - } - } else if (expr instanceof BinaryExpression) { - BinaryExpression binary = (BinaryExpression) expr; - Expression leftExpression = binary.getLeftExpression(); - parserExpressionForFun(udfSet, leftExpression); - Expression rightExpression = binary.getRightExpression(); - parserExpressionForFun(udfSet, rightExpression); - } - } - - /** - * 获取自定义函数类 - * - * @param fun - * @param expressions - * @return - */ - private static UDF getUDF(Function fun, List<Expression> expressions) { - UDF udf; - String funName = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_UNDERSCORE, fun.getName()); - try { - Class dialectClazz = Class.forName("com.mesalab.qgw.model.basic.udf." + CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_UNDERSCORE, fun.getName().toUpperCase())); - Constructor constructor = dialectClazz.getConstructor(UDFElements.class); - UDFElements udfElements = new UDFElements(funName, expressions); - udf = (UDF) constructor.newInstance(udfElements); - } catch (ReflectiveOperationException | RuntimeException e) { - log.error("Custom function conversion instance exception:{}",e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),QGWMessageConst.CUSTOM_FUNCTION_CONVERSION_INSTANCE_EXCEPTION)); - } - return udf; - } - - - - /** - * 执行describe语句 - * - * @param param - * @param statement - * @return - */ - private BaseResult executeDescStatement(QueryProfile param, DescribeStatement statement) { - String tableName = statement.getTable().getName(); - String dbType = param.getDbType(); - if (DBTypeEnum.CLICKHOUSE.getValue().equalsIgnoreCase(dbType)) { - param.setDbType(DBTypeEnum.CLICKHOUSE.getValue()); - param.setDialectDBType(DBTypeEnum.CLICKHOUSE.getValue()); - } else if (DBTypeEnum.DRUID.getValue().equalsIgnoreCase(dbType)) { - param.setDbType(DBTypeEnum.DRUID.getValue()); - param.setDialectDBType(DBTypeEnum.DRUID.getValue()); - param.setQuery("SELECT COLUMN_NAME as name, DATA_TYPE as type FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '" + tableName + "'"); - } else if (DBTypeEnum.HBASE.getValue().equalsIgnoreCase(dbType)) { - param.setDbType(DBTypeEnum.HBASE.getValue()); - param.setDialectDBType(DBTypeEnum.HBASE.getValue()); - param.setQuery(StrFormatter.format("SELECT COLUMN_NAME as \"name\", DATA_TYPE as \"type\" FROM SYSTEM.CATALOG WHERE TABLE_NAME= '{}' AND COLUMN_NAME IS NOT NULL", tableName)); - } else { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), - tableName + ":" + QGWMessageConst.TABLE_NOT_EXIST)); - } - - param.setFormat(QueryFormatEnum.JSON.getValue()); - SQLQuerySource dbQuerySource = new SQLQuerySource(); - dbQuerySource.setSqlBody(param.getQuery()); - param.setDbQuerySource(dbQuerySource); - Dialect dialect = getReflectiveDialectObject(param); - BaseResult baseResult = dataEncapsulationOfSchema(dialect.executeAdministrativeQuery(), param); - - if (!baseResult.isSuccess()) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.CHECK_TABLE)); - } - return baseResult; - } - - private BaseResult executeExplainStatement(QueryProfile param) { - return getDBDialect(param).executeExplainPlan(); - } - - /** - * 执行show语句 - * - * @param param - * @return - */ - private BaseResult executeShowStatement(QueryProfile param) { - return getAllDatabaseTables(param); - } - - - /** - * 获取所有表信息 - * - * @return - */ - private BaseResult getAllDatabaseTables(QueryProfile queryProfile) { - Map<String, Object> dataMap = new LinkedHashMap<>(); - dataMap.put("type", "enum"); - Arrays.stream(DBTypeEnum.values()).forEach(dbTypeEnum -> { - QueryProfile dbParam = new QueryProfile(); - if (dbTypeEnum.getValue().equalsIgnoreCase(DBTypeEnum.CLICKHOUSE.getValue())) { - dbParam.setQuery("show tables"); - dbParam.setDbType(DBTypeEnum.CLICKHOUSE.getValue()); - dbParam.setDialectDBType(SQLHelper.getDialectDBType(DBTypeEnum.CLICKHOUSE.getValue())); - } else if (dbTypeEnum.getValue().equalsIgnoreCase(DBTypeEnum.DRUID.getValue())) { - dbParam.setQuery("SELECT TABLE_NAME AS name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'TABLE'"); - dbParam.setDbType(DBTypeEnum.DRUID.getValue()); - dbParam.setDialectDBType(SQLHelper.getDialectDBType(DBTypeEnum.DRUID.getValue())); - } else { - return; - } - dbParam.setFormat(QueryFormatEnum.JSON.getValue()); - Dialect dialect = getReflectiveDialectObject(dbParam); - BaseResult queryResult = dialect.executeAdministrativeQuery(); - if (queryResult.getStatus() == null || queryResult.getStatus() != ResultStatusEnum.SUCCESS.getCode()) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode() , queryResult.getMessage()); - } - List tables = (ArrayList) queryResult.getData(); - List<String> list = new ArrayList<>(); - Iterator iterator = tables.iterator(); - while (iterator.hasNext()) { - Map<String, String> next = (Map) iterator.next(); - list.add(next.get("name")); - } - BaseResult result = encapsulationShowTables(list, dbParam.getDbType()); - - if (result.getStatus() != null && result.getStatus() == ResultStatusEnum.SUCCESS.getCode()) { - Map dbResultMap = (Map) result.getData(); - - if (StringUtil.isEmpty(dataMap.get("name"))) { - dataMap.put("name", dbResultMap.get("name")); - } else { - dataMap.put("name", Joiner.on(",").join(dataMap.get("name"),dbResultMap.get("name"))); - } - - if (StringUtil.isEmpty(dataMap.get("symbols"))) { - dataMap.put("symbols", dbResultMap.get("symbols")); - } else { - dataMap.put("symbols", Lists.newArrayList(dataMap.get("symbols"), dbResultMap.get("symbols"))); - } - } - - - }); - - return BaseResultGenerator.success("ok", dataMap); - } - - - /** - * 表描述信息结果封装 - * - * @param result - * @param param - * @return - */ - public BaseResult dataEncapsulationOfSchema(BaseResult result, QueryProfile param) { - SchemaBase schema = new SchemaBase(); - schema.setName(SQLHelper.getTableName(param.getQuery()).get(0)); - List<Map> fields = new ArrayList<>(); - List<Map> list = (List<Map>) result.getData(); - for (Map resultMap : list) { - Map schemaMap = Maps.newHashMap(); - schemaMap.put("name", resultMap.get("name")); - schemaMap.put("type", resultMap.get("type")); - fields.add(schemaMap); - } - schema.setFields(fields); - result.setData(schema); - return result; - } - - /** - * 库中包含表表名查询结果处理 - * - * @param tables - * @return - */ - private BaseResult encapsulationShowTables(List tables, String database) { - List<String> list = new ArrayList<>(); - Map<String, Object> date = new LinkedHashMap<>(); - date.put("type", "enum"); - date.put("name", database); - Iterator iterator = tables.iterator(); - while (iterator.hasNext()) { - list.add(String.valueOf(iterator.next())); - } - date.put("symbols", list); - BaseResult<Map> baseResult = BaseResultGenerator.success("ok", null); - baseResult.setData(date); - return baseResult; - } - -} diff --git a/src/main/java/com/mesalab/qgw/service/impl/SQLSyncQueryServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/SQLSyncQueryServiceImpl.java new file mode 100644 index 00000000..0ba15064 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/SQLSyncQueryServiceImpl.java @@ -0,0 +1,666 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.exceptions.ExceptionUtil; +import cn.hutool.core.text.StrFormatter; +import cn.hutool.core.util.BooleanUtil; +import cn.hutool.core.util.StrUtil; +import com.google.common.base.CaseFormat; +import com.google.common.base.Joiner; +import com.google.common.collect.Maps; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.entity.SchemaBase; +import com.mesalab.common.utils.sqlparser.AutoPeriodHelper; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.common.enums.*; +import com.mesalab.common.utils.sqlparser.SQLFunctionUtil; +import com.mesalab.common.utils.sqlparser.SQLHelper; +import com.mesalab.common.utils.sqlparser.SQLSyntaxParserUtil; +import com.mesalab.qgw.dialect.Dialect; +import com.mesalab.qgw.dialect.FederationDialect; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.*; +import com.mesalab.qgw.model.basic.udf.ROLLUP; +import com.mesalab.qgw.model.basic.udf.TIME_FLOOR_WITH_FILL; +import com.mesalab.qgw.model.basic.udf.UDF; +import com.mesalab.qgw.model.basic.udf.UDFElements; +import com.mesalab.qgw.service.SQLSyncQueryService; +import com.mesalab.qgw.service.DatabaseService; +import com.geedgenetworks.utils.StringUtil; +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.DescribeStatement; +import net.sf.jsqlparser.statement.ExplainStatement; +import net.sf.jsqlparser.statement.ShowStatement; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.*; +import net.sf.jsqlparser.util.TablesNamesFinder; +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.lang.reflect.Constructor; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Service("sqlSyncQueryService") +@Slf4j +public class SQLSyncQueryServiceImpl implements SQLSyncQueryService { + private DatabaseService databaseService; + + private final int MAX_PARSER_LEVEL = 5; + private final String PARSER_EXCEPTION_CLASS_NAME = "net.sf.jsqlparser.parser.ParseException:"; + private final String PARSER_EXCEPTION_IDENTIFICATION = "Was expecting one of:"; + private final Pattern PATTERN_ANGLE_BRACKET = Pattern.compile("<(.*?)>", Pattern.CASE_INSENSITIVE); + + + @Override + public BaseResult executeQuery(SQLQueryContext queryContext) { + checkQueryContextParam(queryContext); + parseAndSetConnectorType(queryContext); + Statement statement; + try { + statement = CCJSqlParserUtil.parse(queryContext.getOriginalSQL()); + } catch (JSQLParserException e) { + log.error("Error Parsing SQL: {}", ExceptionUtil.getRootCauseMessage(e)); + String message = getJSQLParserSimpleErrorMessage(e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), message)); + } + BaseResult baseResult; + log.debug("Request SQL Statement: {}", queryContext.getOriginalSQL()); + queryContext.setOriginalSQL(AutoPeriodHelper.buildSqlGranularity(statement).toString()); + if (statement instanceof Select) { + baseResult = executeSelectStatement(queryContext); + } else if (statement instanceof DescribeStatement) { + DescribeStatement describeStatement = (DescribeStatement) statement; + baseResult = executeDescStatement(queryContext, describeStatement); + } else if (statement instanceof ShowStatement) { + baseResult = executeShowStatement(queryContext); + } else if (statement instanceof ExplainStatement) { + baseResult = executeExplainStatement(queryContext); + } else { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), QGWMessageConst.ONLY_SUPPORT_STATEMENT_OPERATIONS_FOR_SELECT)); + } + return baseResult; + } + + private void parseAndSetConnectorType(SQLQueryContext queryContext) { + if (SQLHelper.getTableName(queryContext.getOriginalSQL()).isEmpty()) { + return; + } + String tableName = SQLHelper.getTableName(queryContext.getOriginalSQL()).get(0); + queryContext.setDbEngine(databaseService.getDBEngineByTableName(tableName)); + queryContext.setSqlDialect(SQLHelper.getDialectType(queryContext.getDbEngine())); + } + + private void checkQueryContextParam(SQLQueryContext queryContext) { + if (StringUtil.isBlank(queryContext.getOption())) { + queryContext.setOption(QueryOption.REAL_TIME.getValue()); + } + if (StringUtil.isBlank(queryContext.getFormat())) { + queryContext.setFormat(OutputMode.JSON.getValue()); + } + if (StringUtil.isBlank(queryContext.getOriginalSQL())) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.SQL_MUST_BE_EXISTS)); + } + if (Arrays.stream(QueryOption.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(queryContext.getOption()))) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.OPTION_TYPE_ERROR)); + } + if (Arrays.stream(OutputMode.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(queryContext.getFormat()))) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.FORMAT_TYPE_ERROR)); + } + } + + + /** + * Desc: 简化JSQLParser语法异常信息 + * + * @param e + * @return {@link String} + * @created by wWei + * @date 2021/4/26 9:49 上午 + */ + private String getJSQLParserSimpleErrorMessage(Throwable e) { + String message = Joiner.on(" ").skipNulls().join(ExceptionUtil.getSimpleMessage(e), ExceptionUtil.getRootCauseMessage(e)); + message = message.replace(PARSER_EXCEPTION_CLASS_NAME, StringUtil.EMPTY); + int end = message.indexOf(PARSER_EXCEPTION_IDENTIFICATION); + message = message.substring(0, end < 0 ? message.length() : end); + message = StrUtil.removeAllLineBreaks(message); + Matcher matcher = PATTERN_ANGLE_BRACKET.matcher(message); + if (matcher.find()) { + message = matcher.replaceAll(StringUtil.EMPTY); + } + String[] split = message.split(" "); + StringBuilder stringBuilder = new StringBuilder(message.length()); + for (String c : split) { + if (StringUtil.isBlank(c)) { + continue; + } + stringBuilder.append(" ").append(c); + } + return stringBuilder.toString().trim(); + } + + /** + * 1.设置默认queryID + * 2.为Select操作选择不同的执行动作，包含SQL解析(Syntax-parse)、SQL验证（Syntax-validation）、实时查询(Real-time)、长查询(Long-term) + * + * @param queryContext 请求参数 + * @return BaseResult 执行结果 + */ + private BaseResult executeSelectStatement(SQLQueryContext queryContext) { + if (QueryOption.SYNTAX_PARSE.getValue().equalsIgnoreCase(queryContext.getOption())) { + return executeSyntaxParse(queryContext); + } else if (QueryOption.SYNTAX_VALIDATION.getValue().equalsIgnoreCase(queryContext.getOption())) { + return executeSyntaxValidation(queryContext); + } else if (QueryOption.LONG_TERM.getValue().equalsIgnoreCase(queryContext.getOption())) { + queryContext.setQueryId(databaseService.getCustomQueryId(queryContext.getResultId(), queryContext.getOriginalSQL())); + return executeLongTermQuery(queryContext); + } else { + return executeRealTimeQuery(queryContext); + } + + } + + private BaseResult executeSyntaxParse(SQLQueryContext param) { + return BaseResultGenerator.success(SQLSyntaxParserUtil.syntaxParse(param.getOriginalSQL())); + } + + private BaseResult executeSyntaxValidation(SQLQueryContext queryContext) { + return new FederationDialect(queryContext, getDialect(queryContext).executeSyntaxValidation()).executeSyntaxValidation(); + } + + private BaseResult executeLongTermQuery(SQLQueryContext param) { + return new FederationDialect(param, getDialect(param).executeQuery()).executeQuery(); + } + + + private BaseResult executeRealTimeQuery(SQLQueryContext queryContext) { + if (BooleanUtil.isTrue(queryContext.getSampled())) { + return new FederationDialect(queryContext, getDialect(queryContext).executeSampleQuery()).executeSampleQuery(); + } else { + return new FederationDialect(queryContext, getDialect(queryContext).executeQuery()).executeQuery(); + } + } + + + private Dialect getDialect(SQLQueryContext queryContext) { + buildSelectStatement(queryContext, parserSQLByAst(queryContext.getOriginalSQL())); + return getReflectiveDialectObject(queryContext); + } + + + private Dialect getReflectiveDialectObject(SQLQueryContext queryContext) { + Dialect dialect; + try { + Class dialectClazz = Class.forName("com.mesalab.qgw.dialect." + + CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, queryContext.getDbEngine()) + "Dialect"); + Constructor constructor = dialectClazz.getConstructor(SQLQueryContext.class); + dialect = (Dialect) constructor.newInstance(queryContext); + } catch (ReflectiveOperationException | RuntimeException e) { + log.error("Dialect conversion instance exception:{}", e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); + } + return dialect; + } + + /** + * Parse a nested SQL statement into dbSelectStatement and FederationStatement object + * + * @param sqlQueryContext query context + * @param originalSelectStatement original SQL statement + */ + public void buildSelectStatement(SQLQueryContext sqlQueryContext, SelectStatement originalSelectStatement) { + SelectStatement federationSelectStatement = null; + SelectStatement dbSelectStatement = null; + SelectStatement parentSelectStatement = null; + SelectStatement cursorSelectStatement = originalSelectStatement; + for (int i = 0; i < MAX_PARSER_LEVEL; i++) { + if (hasUDF(cursorSelectStatement)) { + dbSelectStatement = cursorSelectStatement; + break; + } + + if (isEmptySubSelect(cursorSelectStatement)) { + dbSelectStatement = originalSelectStatement; + parentSelectStatement = null; + break; + } else { + parentSelectStatement = cursorSelectStatement; + cursorSelectStatement = cursorSelectStatement.getSubSqlQuerySources().get(0); + if (hasUDF(cursorSelectStatement)) { + dbSelectStatement = cursorSelectStatement; + break; + } + } + } + + if (StringUtil.isNotEmpty(parentSelectStatement)) { + federationSelectStatement = new SelectStatement(); + String tableName = StringUtil.isEmpty(parentSelectStatement.getSubSelect().getAlias()) ? parentSelectStatement.getTableNames().get(0) : parentSelectStatement.getSubSelect().getAlias().getName(); + String replace = originalSelectStatement.getSqlBody().replace(parentSelectStatement.getSubSelect().toString(), tableName); + federationSelectStatement.setSqlBody(replace); + federationSelectStatement.getTableNames().add(0, tableName); + federationSelectStatement.setLimit(originalSelectStatement.getLimit()); + } + + sqlQueryContext.setDbSelectStatement(dbSelectStatement); + sqlQueryContext.setFederationSelectStatement(federationSelectStatement); + } + + private boolean hasUDF(SelectStatement selectStatement) { + return selectStatement != null && selectStatement.getUdfSet() != null && !selectStatement.getUdfSet().isEmpty(); + } + + private boolean isEmptySubSelect(SelectStatement selectStatement) { + return selectStatement == null || StringUtil.isEmpty(selectStatement.getSubSelect()); + } + + + /** + * 解析SQL + * + * @param sql + * @return + */ + private SelectStatement parserSQLByAst(String sql) { + SelectStatement sqlQuerySource = new SelectStatement(); + try { + + Statement statement = CCJSqlParserUtil.parse(sql); + Select selectStatement = null; + if (statement instanceof Select) { + selectStatement = (Select) statement; + } else if (statement instanceof ExplainStatement) { + selectStatement = ((ExplainStatement) statement).getStatement(); + } else { + log.error("Not support DML Parser"); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), QGWMessageConst.SQL_PARSE_ONLY_SUPPORT_SELECT)); + } + + sqlQuerySource.setSqlBody(String.valueOf(selectStatement.getSelectBody())); + List<String> tableList = SQLHelper.getTableName(selectStatement); + tableList.forEach(o -> { + sqlQuerySource.getTableNames().add(o); + }); + sqlQuerySource.setPartitionKey(databaseService.getPartitionKey(sqlQuerySource.getTableNames().get(0))); + SelectBody body = selectStatement.getSelectBody(); + if (body instanceof PlainSelect) { //单条查询 + + PlainSelect select = (PlainSelect) body; + + FromItem fromItem = select.getFromItem(); + sqlQuerySource.setFromItem(fromItem); + sqlQuerySource.setJoins(select.getJoins()); + Expression where = select.getWhere(); + + if (where != null) { + sqlQuerySource.setWhereExpression(where); + } + + List<SelectItem> selectItemsList = select.getSelectItems(); + sqlQuerySource.setSelectItems(selectItemsList); + + for (SelectItem item : selectItemsList) { + if (item instanceof SelectExpressionItem) { + SelectExpressionItem expressionItem = ((SelectExpressionItem) item); + if (StringUtil.isNotEmpty(expressionItem.getAlias())) { + String aliasName = expressionItem.getAlias().getName(); + aliasName = SQLHelper.removeQuotesAndBackticks(aliasName); + sqlQuerySource.getAliasFields().put(aliasName, expressionItem.getExpression().toString()); + } else { + sqlQuerySource.getAliasFields().put(expressionItem.getExpression().toString(), expressionItem.getExpression().toString()); + } + addUDFSet(sqlQuerySource.getUdfSet(), expressionItem.getExpression());//伪代码：后期需要在SQLQuerySource where对象中提取 + + } + } + + GroupByElement groupBy; + if (StringUtil.isNotEmpty(groupBy = select.getGroupBy())) { + sqlQuerySource.setGroupByElement(groupBy); + if (isDrilldown(select, sqlQuerySource.getAliasFields())) { + addRollUpToUDF(sqlQuerySource.getUdfSet(), (Function) groupBy.getGroupByExpressionList().getExpressions().get(0)); + sqlQuerySource.getUdfSet().removeIf(udf -> udf instanceof TIME_FLOOR_WITH_FILL); + } + } + sqlQuerySource.setOrderByElements(select.getOrderByElements()); + FromItem subItem = select.getFromItem(); + + if (subItem instanceof SubSelect) { + SubSelect subSelect = (SubSelect) subItem; + sqlQuerySource.setSubSelect(subSelect); + sqlQuerySource.getSubSqlQuerySources().add(0, parserSQLByAst(String.valueOf(subSelect.getSelectBody()))); + } + Limit limit = select.getLimit(); + if (limit != null) { + + if (StringUtil.isNotEmpty(limit.getOffset())) { + sqlQuerySource.setLimit(StringUtil.setDefaultIfEmpty(limit.getOffset(), 0) + "," + limit.getRowCount()); + } else { + sqlQuerySource.setLimit(String.valueOf(limit.getRowCount())); + } + } + + + } else if (body instanceof SetOperationList) { // 连接查询 + SetOperationList setOperationList = (SetOperationList) body; + List<SelectBody> selects = setOperationList.getSelects(); + //暂时只解析第一个结构，不接受不相同的where + if (StringUtil.isNotEmpty(selects)) { + SelectStatement parseSql = parserSQLByAst(selects.get(0).toString()); + sqlQuerySource.setUdfSet(parseSql.getUdfSet()); + sqlQuerySource.setWhereExpression(parseSql.getWhereExpression()); + sqlQuerySource.setGroupByElement(parseSql.getGroupByElement()); + sqlQuerySource.setAliasFields(parseSql.getAliasFields()); + } + sqlQuerySource.setLimit(SQLHelper.INVALID_LIMIT_DESC); + } else { //其它暂不支持 + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), QGWMessageConst.SQL_PARSE_ONLY_SUPPORT_SELECT)); + } + + + } catch (JSQLParserException | RuntimeException e) { + log.error("sqlParser error: {}", e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); + + } + + return sqlQuerySource; + } + + private static boolean isDrilldown(PlainSelect select, Map<String, String> aliasFields) { + if (select.getGroupBy() == null) { + return false; + } + List<Expression> groupByExpressionsList = select.getGroupBy().getGroupByExpressionList().getExpressions(); + if (groupByExpressionsList.size() != 1) { + return false; + } + Expression groupByExpression = groupByExpressionsList.get(0); + if (!(groupByExpression instanceof Function)) { + return false; + } + Function groupByFunction = (Function) groupByExpression; + if (!groupByFunction.getName().equalsIgnoreCase(SQLFunctionUtil.ROLLUP) || groupByFunction.getParameters() == null || groupByFunction.getParameters().getExpressions().size() != 2) { + return false; + } + if (select.getOrderByElements() == null || select.getOrderByElements().size() != 1 || select.getOrderByElements().get(0).isAsc()) { + return false; + } + Expression orderByExpr = select.getOrderByElements().get(0).getExpression(); + if (orderByExpr instanceof Function) { + Function orderByFunction = (Function) orderByExpr; + if (orderByFunction.isDistinct()) { + return false; + } + if (!"count".equalsIgnoreCase(orderByFunction.getName()) && !"sum".equalsIgnoreCase(orderByFunction.getName())) { + return false; + } + } else { + String orderByKey = SQLHelper.removeQuotesAndBackticks(orderByExpr.toString()); + if (!aliasFields.containsKey(orderByKey)) { + return false; + } + String orderByExprTmp = String.valueOf(aliasFields.get(orderByKey)).toLowerCase(); + if (orderByExprTmp.startsWith("count(distinct")) { + return false; + } + if (!orderByExprTmp.startsWith("count(") && !orderByExprTmp.startsWith("sum(")) { + return false; + } + } + List<Expression> expressions = groupByFunction.getParameters().getExpressions(); + for (Expression expression : expressions) { + String rollupArg = SQLHelper.removeQuotesAndBackticks(expression.toString()); + if (!aliasFields.containsKey(rollupArg) && !aliasFields.containsValue(rollupArg)) { + return false; + } + } + return true; + } + + private static void addRollUpToUDF(Set<UDF> udfSet, Function fun) { + List<Expression> expressions = fun.getParameters().getExpressions(); + UDFElements udfElements = new UDFElements(fun.getName(), expressions); + ROLLUP rollup = new ROLLUP(udfElements); + udfSet.add(rollup); + } + + /** + * 自定义函数封装 + * + * @param udfSet + * @param expr + */ + private static void addUDFSet(Set<UDF> udfSet, Expression expr) { + parserExpressionForFun(udfSet, expr); + } + + private static void parserExpressionForFun(Set<UDF> udfSet, Expression expr) { + if (expr instanceof Function) { + Function fun = (Function) expr; + if (SQLFunctionUtil.federationUDFFunctions.containsKey(fun.getName().toUpperCase())) { + List<Expression> expressions = fun.getParameters().getExpressions(); + UDF udf = getUDF(fun, expressions); + udfSet.add(udf); + } + if (fun.getParameters() == null) { + return; + } + for (Expression expression : fun.getParameters().getExpressions()) { + parserExpressionForFun(udfSet, expression); + } + } else if (expr instanceof BinaryExpression) { + BinaryExpression binary = (BinaryExpression) expr; + Expression leftExpression = binary.getLeftExpression(); + parserExpressionForFun(udfSet, leftExpression); + Expression rightExpression = binary.getRightExpression(); + parserExpressionForFun(udfSet, rightExpression); + } + } + + /** + * 获取自定义函数类 + * + * @param fun + * @param expressions + * @return + */ + private static UDF getUDF(Function fun, List<Expression> expressions) { + UDF udf; + String funName = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_UNDERSCORE, fun.getName()); + try { + Class dialectClazz = Class.forName("com.mesalab.qgw.model.basic.udf." + CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_UNDERSCORE, fun.getName().toUpperCase())); + Constructor constructor = dialectClazz.getConstructor(UDFElements.class); + UDFElements udfElements = new UDFElements(funName, expressions); + udf = (UDF) constructor.newInstance(udfElements); + } catch (ReflectiveOperationException | RuntimeException e) { + log.error("Custom function conversion instance exception:{}", e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), QGWMessageConst.CUSTOM_FUNCTION_CONVERSION_INSTANCE_EXCEPTION)); + } + return udf; + } + + + /** + * 执行describe语句 + * + * @param queryContext + * @param statement + * @return + */ + private BaseResult executeDescStatement(SQLQueryContext queryContext, DescribeStatement statement) { + String tableName = statement.getTable().getName(); + String dbEngineType = queryContext.getDbEngine(); + if (DBEngineType.CLICKHOUSE.getValue().equalsIgnoreCase(dbEngineType)) { + queryContext.setDbEngine(DBEngineType.CLICKHOUSE.getValue()); + queryContext.setSqlDialect(DBEngineType.CLICKHOUSE.getValue()); + } else if (DBEngineType.DRUID.getValue().equalsIgnoreCase(dbEngineType)) { + queryContext.setDbEngine(DBEngineType.DRUID.getValue()); + queryContext.setSqlDialect(DBEngineType.DRUID.getValue()); + queryContext.setOriginalSQL("SELECT COLUMN_NAME as name, DATA_TYPE as type FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '" + tableName + "'"); + } else if (DBEngineType.HBASE.getValue().equalsIgnoreCase(dbEngineType)) { + queryContext.setDbEngine(DBEngineType.HBASE.getValue()); + queryContext.setSqlDialect(DBEngineType.HBASE.getValue()); + queryContext.setOriginalSQL(StrFormatter.format("SELECT COLUMN_NAME as \"name\", DATA_TYPE as \"type\" FROM SYSTEM.CATALOG WHERE TABLE_NAME= '{}' AND COLUMN_NAME IS NOT NULL", tableName)); + } else { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), + tableName + ":" + QGWMessageConst.TABLE_NOT_EXIST)); + } + + queryContext.setFormat(OutputMode.JSON.getValue()); + SelectStatement dbSelectStatement = new SelectStatement(); + dbSelectStatement.setSqlBody(queryContext.getOriginalSQL()); + queryContext.setDbSelectStatement(dbSelectStatement); + Dialect dialect = getReflectiveDialectObject(queryContext); + BaseResult baseResult = dataEncapsulationOfSchema(dialect.executeAdministrativeQuery(), queryContext); + + if (!baseResult.isSuccess()) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.CHECK_TABLE)); + } + return baseResult; + } + + private BaseResult executeExplainStatement(SQLQueryContext param) { + return getDialect(param).executeExplainPlan(); + } + + /** + * 执行show语句 + * + * @param param + * @return + */ + private BaseResult executeShowStatement(SQLQueryContext param) { + return getAllDatabaseTables(param); + } + + + /** + * 获取所有表信息 + * + * @return + */ + private BaseResult getAllDatabaseTables(SQLQueryContext queryContext) { + Map<String, Object> dataMap = new LinkedHashMap<>(); + dataMap.put("type", "enum"); + Arrays.stream(DBEngineType.values()).forEach(dbTypeEnum -> { + SQLQueryContext dbParam = new SQLQueryContext(); + if (dbTypeEnum.getValue().equalsIgnoreCase(DBEngineType.CLICKHOUSE.getValue())) { + dbParam.setOriginalSQL("show tables"); + dbParam.setDbEngine(DBEngineType.CLICKHOUSE.getValue()); + dbParam.setSqlDialect(SQLHelper.getDialectType(DBEngineType.CLICKHOUSE.getValue())); + } else if (dbTypeEnum.getValue().equalsIgnoreCase(DBEngineType.DRUID.getValue())) { + dbParam.setOriginalSQL("SELECT TABLE_NAME AS name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'TABLE'"); + dbParam.setDbEngine(DBEngineType.DRUID.getValue()); + dbParam.setSqlDialect(SQLHelper.getDialectType(DBEngineType.DRUID.getValue())); + } else { + return; + } + dbParam.setFormat(OutputMode.JSON.getValue()); + Dialect dialect = getReflectiveDialectObject(dbParam); + BaseResult queryResult = dialect.executeAdministrativeQuery(); + if (queryResult.getStatus() == null || queryResult.getStatus() != HttpStatusCodeEnum.SUCCESS.getCode()) { + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), queryResult.getMessage()); + } + List tables = (ArrayList) queryResult.getData(); + List<String> list = new ArrayList<>(); + Iterator iterator = tables.iterator(); + while (iterator.hasNext()) { + Map<String, String> next = (Map) iterator.next(); + list.add(next.get("name")); + } + BaseResult result = encapsulationShowTables(list, dbParam.getDbEngine()); + + if (result.getStatus() != null && result.getStatus() == HttpStatusCodeEnum.SUCCESS.getCode()) { + Map dbResultMap = (Map) result.getData(); + + if (StringUtil.isEmpty(dataMap.get("name"))) { + dataMap.put("name", dbResultMap.get("name")); + } else { + dataMap.put("name", Joiner.on(",").join(dataMap.get("name"), dbResultMap.get("name"))); + } + + if (StringUtil.isEmpty(dataMap.get("symbols"))) { + dataMap.put("symbols", dbResultMap.get("symbols")); + } else { + dataMap.put("symbols", Lists.newArrayList(dataMap.get("symbols"), dbResultMap.get("symbols"))); + } + } + + + }); + + return BaseResultGenerator.success("ok", dataMap); + } + + + /** + * 表描述信息结果封装 + * + * @param result + * @param param + * @return + */ + public BaseResult dataEncapsulationOfSchema(BaseResult result, SQLQueryContext param) { + SchemaBase schema = new SchemaBase(); + schema.setName(SQLHelper.getTableName(param.getOriginalSQL()).get(0)); + List<Map> fields = new ArrayList<>(); + List<Map> list = (List<Map>) result.getData(); + for (Map resultMap : list) { + Map schemaMap = Maps.newHashMap(); + schemaMap.put("name", resultMap.get("name")); + schemaMap.put("type", resultMap.get("type")); + fields.add(schemaMap); + } + schema.setFields(fields); + result.setData(schema); + return result; + } + + /** + * 库中包含表表名查询结果处理 + * + * @param tables + * @return + */ + private BaseResult encapsulationShowTables(List tables, String database) { + List<String> list = new ArrayList<>(); + Map<String, Object> date = new LinkedHashMap<>(); + date.put("type", "enum"); + date.put("name", database); + Iterator iterator = tables.iterator(); + while (iterator.hasNext()) { + list.add(String.valueOf(iterator.next())); + } + date.put("symbols", list); + BaseResult<Map> baseResult = BaseResultGenerator.success("ok", null); + baseResult.setData(date); + return baseResult; + } + + @Autowired + public void setDatabaseService(DatabaseService databaseService) { + this.databaseService = databaseService; + } + +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java deleted file mode 100644 index 308ad06d..00000000 --- a/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java +++ /dev/null @@ -1,650 +0,0 @@ -package com.mesalab.qgw.service.impl; - -import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.util.StrUtil; -import cn.hutool.core.util.URLUtil; -import cn.hutool.crypto.digest.DigestUtil; -import cn.hutool.json.JSONUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.alibaba.fastjson2.JSON; -import com.google.common.base.CaseFormat; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.jayway.jsonpath.JsonPath; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.*; -import com.mesalab.common.utils.sqlparser.SQLHelper; -import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.qgw.model.basic.ClickHouseHttpSource; -import com.mesalab.qgw.model.basic.HttpConfig; -import com.mesalab.qgw.dialect.Dialect; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.model.basic.JobAdminHttpSource; -import com.mesalab.qgw.model.job.EncryptionInfo; -import com.mesalab.qgw.model.job.ExecutorParam; -import com.mesalab.qgw.model.job.StorageDeletionInfo; -import com.mesalab.qgw.model.job.XxlJobInfo; -import com.mesalab.qgw.service.QueryService; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.qgw.service.SystemService; -import com.mesalab.services.common.property.SqlPropertySourceFactory; -import com.geedgenetworks.utils.DateUtils; -import com.geedgenetworks.utils.Encodes; -import com.geedgenetworks.utils.StringUtil; -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; -import org.jasypt.util.text.BasicTextEncryptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cglib.beans.BeanMap; -import org.springframework.context.EnvironmentAware; -import org.springframework.context.annotation.PropertySource; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; -import java.lang.reflect.AccessibleObject; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -@Service("systemService") -@PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) -public class SystemServiceImpl implements SystemService, EnvironmentAware { - private static final Log log = LogFactory.get(); - private static Pattern pTTL = Pattern.compile(".*toIntervalSecond\\((\\d+)\\)", Pattern.CASE_INSENSITIVE); - private static final String trafficPort = "8123"; - private Environment env; - @Autowired - private QueryService queryService; - @Autowired - private JobAdminHttpSource jobAdminHttpSource; - @Autowired - private HttpClientService httpClientService; - @Resource(name = "metadataService") - private MetadataService metadataService; - @Autowired - HttpConfig httpConfig; - @Autowired - ClickHouseHttpSource clickHouseHttpSource; - - public final static String queryIdSeparator = ":"; - - private Map<String, String> headers = Maps.newHashMap(); - - @Override - public BaseResult getStorageQuota() { - - //封装sql进行查询：Analytic Logs、Files、Traffic Logs - QueryProfile param = new QueryProfile(); - param.setQuery(env.getProperty("SYSTEM_STORAGE_QUOTA")); - BaseResult result = queryService.executeQuery(param); - return resetBaseResult(result, "type"); - } - - @Override - public BaseResult dailyTrendOfStorage(String searchStartTime, String searchEndTime) { - - Date currentDate = DateUtils.convertStringToDate(DateUtils.getCurrentDate(), DateUtils.YYYY_MM_DD); - - if (StringUtil.isBlank(searchStartTime) && StringUtil.isBlank(searchEndTime)) { - searchStartTime = DateUtils.getFormatDate(DateUtils.getSomeDate(currentDate, -7), DateUtils.YYYY_MM_DD_HH24_MM_SS); - searchEndTime = DateUtils.getFormatDate(currentDate, DateUtils.YYYY_MM_DD_HH24_MM_SS); - } - - if (StringUtil.isNotBlank(searchStartTime) && StringUtil.isNotBlank(searchEndTime)) { - QueryProfile param = new QueryProfile(); - String sql = String.format(env.getProperty("SYSTEM_DAILY_TREND_OF_STORAGE"), searchStartTime, searchEndTime); - param.setQuery(sql); - BaseResult result = queryService.executeQuery(param); - List<Map> data = (List<Map>) result.getData(); - data.forEach(o -> o.put("type", LogType.getLabelByValue(String.valueOf(o.get("type"))))); - return result; - } else { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.START_TIME_AND_END_TIME_NOT_NULL)); - } - } - - private BaseResult resetBaseResult(BaseResult result, String alias) { - if (!result.isSuccess()) { - return result; - } - List<Map> data = (List<Map>) result.getData(); - for (Map map : data){ - String logType = LogType.getLabelByValue(String.valueOf(map.get(alias))); - BaseResult deleteStorageStatus = getDeleteStorageStatus(String.valueOf(map.get(alias))); - List<StorageDeletionInfo> data1 = (List<StorageDeletionInfo>) deleteStorageStatus.getData(); - map.put(alias, logType); - map.put("max_days",data1.get(0).getMaxDays()); - map.put("default_max_days",data1.get(0).getDefaultMaxDays()); - } - return result; - } - - @Override - public BaseResult deleteStorage(List<StorageDeletionInfo> list) { - - if (!logTypeValid(list)) { - return BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"Match failed, please check log type!")); - } - - preProcessOfLogType(list); - - if (jobIsBusy(list)) { - return BaseResultGenerator.failure(ResultStatusEnum.LOCKED.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"The task is busy, please try later!")); - } - - BaseResult baseResult = null; - for (StorageDeletionInfo info : list) { - baseResult = executeDeleteStorageJob(info); - } - return baseResult; - } - - @Override - public BaseResult getDeleteStorageStatus(String logType) { - setCookie(); - List<StorageDeletionInfo> list = new ArrayList<>(); - if (StringUtil.isBlank(logType)) { - - StorageDeletionInfo trafficInfo = getStorageDeletionInfoByHandler(LogType.TRAFFIC_LOGS.getValue(), JobHandlerEnum.DELETE_TRAFFIC_DATA_JOB_HANDLER.getValue()); - list.add(trafficInfo); - - StorageDeletionInfo reportInfo = getStorageDeletionInfoByHandler(LogType.REPORT_AND_METRICS.getValue(), JobHandlerEnum.DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue()); - list.add(reportInfo); - - StorageDeletionInfo fileInfo = getStorageDeletionInfoByHandler(LogType.FILES.getValue(), JobHandlerEnum.DELETE_FILES_JOB_HANDLER.getValue()); - list.add(fileInfo); - } else if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { - StorageDeletionInfo trafficInfo = getStorageDeletionInfoByHandler(logType, JobHandlerEnum.DELETE_TRAFFIC_DATA_JOB_HANDLER.getValue()); - list.add(trafficInfo); - } else if (LogType.REPORT_AND_METRICS.getValue().equalsIgnoreCase(logType)) { - StorageDeletionInfo reportInfo = getStorageDeletionInfoByHandler(logType, JobHandlerEnum.DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue()); - list.add(reportInfo); - } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { - StorageDeletionInfo reportInfo = getStorageDeletionInfoByHandler(logType, JobHandlerEnum.DELETE_FILES_JOB_HANDLER.getValue()); - list.add(reportInfo); - } else { - return BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), "No matching log type: " + logType); - } - list.forEach(o -> o.setType(LogType.getLabelByValue(o.getType()))); - return BaseResultGenerator.success("ok", list); - } - - - @Override - public String getCustomQueryId(String resultId, String query) { - String tableName = SQLHelper.getTableName(query).get(0); - String dbType = metadataService.getDBTypeByTableName(tableName); - return DigestUtil.md5Hex(dbType) + queryIdSeparator + DigestUtil.md5Hex(resultId + query.trim()); - } - - - @Override - public BaseResult getCiphertext(EncryptionInfo param) { - BaseResult baseResult; - BasicTextEncryptor textEncryptor = new BasicTextEncryptor(); - textEncryptor.setPassword(param.getSalt()); - String encrypt = textEncryptor.encrypt(param.getPassword()); - Map dataMap = new HashMap(); - dataMap.put("password", param.getPassword()); - dataMap.put("encrypted_password", encrypt); - baseResult = BaseResultGenerator.success("ok", Lists.newArrayList(dataMap)); - return baseResult; - } - - - - - private boolean logTypeValid(List<StorageDeletionInfo> list) { - for (StorageDeletionInfo info : list) { - String value = LogType.getValueByLabel(info.getType()); - if (StringUtil.isEmpty(value)) { - return false; - } - } - return true; - } - - /** - * @Description 涉及对ALL、LogType(value，label)处理 - * @Param list: - * @return: java.util.List<com.mesalab.qgw.model.job.StorageDeletionInfo> - * @Date: 2021/1/4 11:36 上午 - * @Created by wWei - */ - private List<StorageDeletionInfo> preProcessOfLogType(List<StorageDeletionInfo> list) { - if (list.size() == 1){ - StorageDeletionInfo deletionInfo = list.get(0); - Integer maxDays = deletionInfo.getMaxDays(); - Integer defaultMaxDays = deletionInfo.getDefaultMaxDays(); - if (LogType.ALL.getValue().equalsIgnoreCase(list.get(0).getType())){ - list.clear(); - list.add(new StorageDeletionInfo(LogType.TRAFFIC_LOGS.getValue(), maxDays, defaultMaxDays)); - list.add(new StorageDeletionInfo(LogType.REPORT_AND_METRICS.getValue(), maxDays, defaultMaxDays)); - list.add(new StorageDeletionInfo(LogType.FILES.getValue(), maxDays, defaultMaxDays)); - return list; - }else if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(list.get(0).getType()) | - LogType.FILES.getValue().equalsIgnoreCase(list.get(0).getType()) | - LogType.REPORT_AND_METRICS.getLabel().equalsIgnoreCase(list.get(0).getType())) { - list.clear(); - list.add(new StorageDeletionInfo(LogType.getValueByLabel(deletionInfo.getType()), maxDays, defaultMaxDays)); - return list; - } - } - list.forEach(o -> o.setType(LogType.getValueByLabel(o.getType()))); - return list; - } - - private boolean jobIsBusy(List<StorageDeletionInfo> list) { - setCookie(); - for (StorageDeletionInfo info : list) { - - String handler = getDeletePartHandlerByLogType(info.getType()); - if (jobIsBusyByHandler(handler)) { - return true; - } - handler = getDeleteAllHandlerByLogType(info.getType()); - if (jobIsBusyByHandler(handler)) { - return true; - } - } - return false; - } - - private boolean jobIsBusyByHandler(String handler) { - Map dataByHandler = getDataByHandler(handler); - String id = String.valueOf(dataByHandler.get("id")); - BaseResult result = queryJobStatusByJobId(Integer.parseInt(id)); - if (result.getStatus().equals(ResultStatusEnum.LOCKED.getCode())) { - return true; - } else if (!result.getStatus().equals(ResultStatusEnum.SUCCESS.getCode())) { - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),result.getMessage())); - } - return false; - } - - - /** - * 执行日志删除任务: 调用调度任务任务 - * - * @param info - * @return - */ - private BaseResult executeDeleteStorageJob(StorageDeletionInfo info) { - - BaseResult baseResult; - String jobHandler; - String logType = info.getType(); - ExecutorParam executorParam = new ExecutorParam(); - if (0 == info.getMaxDays()) { - jobHandler = getDeleteAllHandlerByLogType(logType); - Map dataByHandler = getDataByHandler(jobHandler); - - StorageDeletionInfo storageDeletionInfoByHandler = getStorageDeletionInfoByHandler(logType, getDeletePartHandlerByLogType(logType)); - executorParam.setMaxDays(storageDeletionInfoByHandler.getMaxDays()); - XxlJobInfo xxlJobInfo = setXxlJobInfoParam(dataByHandler, executorParam); - baseResult = executeTriggerAndUpdate(xxlJobInfo); - } else { - jobHandler = getDeletePartHandlerByLogType(info.getType()); - executorParam.setMaxDays(info.getMaxDays()); - Map oldData = getDataByHandler(jobHandler); - XxlJobInfo newData = setXxlJobInfoParam(oldData, executorParam); - baseResult = executeManageJob("update", newData); - if (baseResult.isSuccess() && LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(info.getType())) { - updateSchema(info, oldData); - } - if (baseResult.isSuccess()) { - baseResult = executeManageJob("trigger", newData); - } - } - if (!baseResult.getStatus().equals(ResultStatusEnum.SUCCESS.getCode())) { - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),QGWMessageConst.SCHEDULED_TASK_ERROR)); - } - return baseResult; - } - - private void updateSchema(StorageDeletionInfo info, Map dataByHandler) { - Map<String, Map> previous = Maps.newHashMap(); - try { - Map schemaInfo = metadataService.getSchemaInfo(MetadataTypeEnum.TABLES.getValue(), clickHouseHttpSource.getDbName(), false); - Object tables = schemaInfo.get("symbols"); - if (StringUtil.isEmpty(tables)) { - return; - } - long ttl = 24 * 60 * 60 * info.getMaxDays(); - List<String> list = (List) tables; - for (String tableName : list) { - Map schemaMap = metadataService.getSchemaInfo(MetadataTypeEnum.FIELDS.getValue(), tableName, false); - List<Object> schemaDocTTL = JsonPath.read(schemaMap, "$.[?(@.doc.ttl != null)].doc.ttl"); - List<Object> fieldDocTTL = JsonPath.read(schemaMap, "$.fields[?(@.doc.ttl != null)].doc.ttl"); - if (schemaDocTTL.isEmpty() && fieldDocTTL.isEmpty()) { - continue; - } - previous.put(tableName, metadataService.getSchemaInfo(MetadataTypeEnum.FIELDS.getValue(), tableName, false)); - Object schemaDoc = schemaMap.get("doc"); - Map<String, Object> map = Maps.newHashMap(); - if (StringUtil.isNotEmpty(schemaDoc)) { - map = (Map<String, Object>) schemaDoc; - } - if (StringUtil.isNotEmpty(map.get("ttl")) && ttl < Long.parseLong(map.get("ttl").toString())) { - map.put("ttl", ttl); - schemaMap.put("doc", map); - } - List<Map<String, Object>> fields = JsonPath.read(schemaMap, "$.fields"); - List<String> indexKey = metadataService.getIndexKey(tableName); - for (Map<String, Object> field : fields) { - Object doc = field.get("doc"); - if (StringUtil.isEmpty(doc)) { - doc = Maps.newHashMap(); - } - Map<String, Object> fieldDoc = (Map<String, Object>) doc; - if (indexKey.contains(field.get("name").toString())) { - fieldDoc.put("ttl", null); - field.put("doc", fieldDoc); - continue; - } - if (StringUtil.isNotEmpty(fieldDoc.get("ttl")) && ttl < Long.parseLong(fieldDoc.get("ttl").toString())) { - fieldDoc.put("ttl", ttl); - field.put("doc", fieldDoc); - } - } - metadataService.updateSchema(tableName, schemaMap); - } - } catch (RuntimeException ex) { - for (String tableName : previous.keySet()) { - metadataService.updateSchema(tableName, previous.get(tableName)); - } - executeManageJob("update", mapToBean(dataByHandler, XxlJobInfo.class)); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),"up schema error in storage set ", ex.getMessage())); - } - } - - private String getDeleteAllHandlerByLogType(String logType) { - String jobHandler = StringUtil.EMPTY; - if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_ALL_TRAFFIC_DATA_JOB_HANDLER.getValue(); - } else if (LogType.REPORT_AND_METRICS.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_ALL_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue(); - } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_ALL_FILES_JOB_HANDLER.getValue(); - } - return jobHandler; - } - - private String getDeletePartHandlerByLogType(String logType) { - String jobHandler = StringUtil.EMPTY; - if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_TRAFFIC_DATA_JOB_HANDLER.getValue(); - } else if (LogType.REPORT_AND_METRICS.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue(); - } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_FILES_JOB_HANDLER.getValue(); - } - return jobHandler; - } - - /** - * 通过handler获取数据配额设置状态 - * - * @param logType - * @param jobHandlerValue - * @return - */ - private StorageDeletionInfo getStorageDeletionInfoByHandler(String logType, String jobHandlerValue) { - Map trafficDate = getDataByHandler(jobHandlerValue); - XxlJobInfo xxlJobInfo = mapToBean(trafficDate, XxlJobInfo.class); - StorageDeletionInfo executorParam = JSON.parseObject(StrUtil.toUnderlineCase(xxlJobInfo.getExecutorParam()), StorageDeletionInfo.class); - executorParam.setType(logType); - return executorParam; - } - - /** - * 触发执行器并更新任务 - * - * @param xxlJobInfo - * @return - */ - private BaseResult executeTriggerAndUpdate(XxlJobInfo xxlJobInfo) { - BaseResult baseResult; - BaseResult resultExecute = executeManageJob("trigger", xxlJobInfo); - if (resultExecute.getStatus().equals(ResultStatusEnum.SUCCESS.getCode())) { - BaseResult resultUpdate = executeManageJob("update", xxlJobInfo); - if (resultUpdate.getStatus().equals(ResultStatusEnum.SUCCESS.getCode())) { - baseResult = BaseResultGenerator.success("ok", null); - } else { - baseResult = BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),"Update task failed:" + xxlJobInfo.getExecutorHandler())); - } - } else { - baseResult = BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),"Failed to execute task:" + xxlJobInfo.getExecutorHandler())); - } - return baseResult; - } - - /** - * 查询任务状态 - * - * @param jobId - * @return - */ - private BaseResult queryJobStatusByJobId(int jobId) { - BaseResult baseResult = null; - StringBuilder url = new StringBuilder(jobAdminHttpSource.getUrl()).append("/jobinfo/jobBeat?jobId=").append(jobId); - Map<String, String> resultMap = httpClientService.httpGet(url.toString(), headers, httpConfig.getServerResponseTimeOut()); - baseResult = resultEncapsulationOfJob(resultMap); - return baseResult; - } - - /** - * 执行调度任务结果封装 - * - * @param resultMap - * @return - */ - private BaseResult resultEncapsulationOfJob(Map<String, String> resultMap) { - BaseResult baseResult; - if (StringUtil.isEmpty(resultMap)) { - baseResult = BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),"The service is busy, please contact the scheduling platform!")); - } else { - if (resultMap.get("status").equals(String.valueOf(ResultStatusEnum.SUCCESS.getCode()))) { - Map result = JSON.parseObject(resultMap.get("result"), Map.class); - if (result.get("code").equals(ResultStatusEnum.SUCCESS.getCode())) { - baseResult = BaseResultGenerator.success("ok", null); - } else if (result.get("code").equals(ResultStatusEnum.LOCKED.getCode())) { - baseResult = BaseResultGenerator.failure(ResultStatusEnum.LOCKED.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), "Clear Task is Running.")); - } else { - baseResult = BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), result.get("msg"))); - } - } else { - baseResult = BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), "Clear Task Failed.")); - } - } - return baseResult; - } - - - /** - * 执行调度平台任务更新 - * - * @param executeType - * @param xxlJobInfo - * @return - */ - private BaseResult executeManageJob(String executeType, XxlJobInfo xxlJobInfo) { - BaseResult baseResult = null; - String params = getUrlParamsByMap(getObjectToMap(xxlJobInfo)); - String url = String.format("%s/jobinfo/%s/?%s", jobAdminHttpSource.getUrl(), executeType, params); - Map<String, String> resultMap = httpClientService.httpGet(url, headers, httpConfig.getServerResponseTimeOut()); - log.warn("请求调度任务" + executeType + "接口" + url); - baseResult = resultEncapsulationOfJob(resultMap); - return baseResult; - } - - - /** - * 获取调度平台Cookie - * - * @return - */ - public void setCookie() { - StringBuilder url = new StringBuilder(jobAdminHttpSource.getUrl()); - headers.put("Content-Type", "application/json"); - String urlParamsByMap = getUrlParamsByMap(getObjectToMap(jobAdminHttpSource)); - int socketTimeOut = httpConfig.getServerResponseTimeOut(); - Map httpPostResponseHeads = httpClientService.getHttpPostResponseHeads(url + "/login?" + urlParamsByMap, headers, socketTimeOut); - String cookie = String.valueOf(httpPostResponseHeads.get("SET-COOKIE")); - headers.put("Cookie", cookie); - } - - /** - * 通过handler Value获取jobInfo数据 - * - * @param handlerValue - * @return - */ - private Map getDataByHandler(String handlerValue) { - StringBuilder url = new StringBuilder(jobAdminHttpSource.getUrl()). - append("/jobinfo/pageList?jobGroup=-1&triggerStatus=-1&executorHandler="). - append(handlerValue); - Map<String, String> resultPageList = httpClientService.httpGet(url.toString(), headers, httpConfig.getServerResponseTimeOut()); - if (StringUtil.isNotEmpty(resultPageList) && resultPageList.get("status").equals(String.valueOf(ResultStatusEnum.SUCCESS.getCode()))) { - Map<String, Object> maps = JSON.parseObject(resultPageList.get("result"), Map.class); - if (StringUtil.isEmpty(maps)) { - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),"Get or package result exception: " + JSON.toJSONString(resultPageList))); - } - List<Map> data = (List) maps.get("data"); - if (data.size() >= 1) { - return data.get(0); - } else { - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),"The scheduled task has no task information matching executorHandler (" + handlerValue + ")")); - } - } - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(),"Get scheduled task exception: " + JSON.toJSONString(resultPageList))); - } - - /** - * 更新jobInfo - * - * @param data - * @param executorParam - * @return - */ - private XxlJobInfo setXxlJobInfoParam(Map data, ExecutorParam executorParam) { - XxlJobInfo xxlJobInfo = mapToBean(data, XxlJobInfo.class); - ExecutorParam executor = JSON.parseObject(xxlJobInfo.getExecutorParam(), ExecutorParam.class); - executorParam.setDefaultMaxDays(executor.getDefaultMaxDays()); - xxlJobInfo.setExecutorParam(JSON.toJSONString(executorParam)); - return xxlJobInfo; - } - - public static String getUrlParamsByMap(Map<String, Object> map) { - if (map == null) { - return ""; - } - StringBuffer sb = new StringBuffer(); - for (Map.Entry<String, Object> entry : map.entrySet()) { - sb.append(entry.getKey() + "=" + Encodes.urlEncode(String.valueOf(entry.getValue()))); - sb.append("&"); - } - String s = sb.toString(); - if (s.endsWith("&")) { - s = StringUtil.substringBeforeLast(s, "&"); - } - return s; - } - - public static <T> T mapToBean(Map<String, Object> map, Class<T> clazz) { - T bean = null; - try { - bean = clazz.newInstance(); - } catch (InstantiationException e) { - log.error("Instantiation Exception: ", e); - } catch (IllegalAccessException e) { - log.error("Illegal Access Exception: ", e); - } - BeanMap beanMap = BeanMap.create(bean); - beanMap.putAll(map); - return bean; - } - - public static Map<String, Object> getObjectToMap(Object obj) { - Map<String, Object> map = new LinkedHashMap<String, Object>(); - Class<?> clazz = obj.getClass(); - Field[] declaredFields = clazz.getDeclaredFields(); - AccessibleObject.setAccessible(declaredFields, true); - for (Field field : declaredFields) { - String fieldName = field.getName(); - Object value = null; - try { - value = field.get(obj); - } catch (IllegalAccessException e) { - log.error("Illegal Access Exception: ", e); - } - if (value == null) { - value = ""; - } - map.put(fieldName, value); - } - return map; - } - - @Override - public void setEnvironment(Environment environment) { - this.env = environment; - } - - - @Getter - @AllArgsConstructor - enum LogType { - TRAFFIC_LOGS("Traffic Logs", "Traffic Logs"), - REPORT_AND_METRICS("Report and Metrics", "Metrics"), - FILES("Files", "Files"), - ALL("ALL", "ALL"); - private final String value; - private final String label; - - - public static String getValueByLabel(String label) { - for (LogType enums : LogType.values()) { - if (enums.getLabel().equals(label)) { - return enums.getValue(); - } - } - return ""; - } - - public static String getLabelByValue(String value) { - for (LogType enums : LogType.values()) { - if (enums.getValue().equals(value)) { - return enums.getLabel(); - } - } - return ""; - } - } -} diff --git a/src/main/java/com/mesalab/qgw/service/impl/TrafficSpectrumDslServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/TrafficSpectrumDslServiceImpl.java new file mode 100644 index 00000000..a615a2a4 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/TrafficSpectrumDslServiceImpl.java @@ -0,0 +1,588 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.lang.UUID; +import cn.hutool.core.util.NumberUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.crypto.digest.DigestUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.geedgenetworks.utils.IPUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.enums.QueryOption; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.constant.dsl.TrafficSpectrumConstants; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.qgw.model.basic.DSLQueryContext; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import com.mesalab.qgw.model.basic.EngineConfigSource; +import com.mesalab.qgw.model.basic.SQLQueryContext; +import com.mesalab.qgw.service.DatabaseService; +import com.mesalab.qgw.service.SQLSyncQueryService; +import com.mesalab.qgw.service.TrafficSpectrumDslService; +import com.mesalab.services.common.property.SqlPropertySourceFactory; +import com.mesalab.services.configuration.JobConfig; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.util.*; +import java.util.stream.Collectors; + +/** + * TODO + * + * @Classname TrafficSpectrumDslServiceImpl + * @Date 2024/5/10 15:01 + * @Author wWei + */ +@Service +@PropertySource(value = "classpath:dsl-sql-template.sql", factory = SqlPropertySourceFactory.class) +public class TrafficSpectrumDslServiceImpl implements TrafficSpectrumDslService { + + private static final Log log = LogFactory.get(); + private Environment environment; + private DatabaseService databaseService; + private SQLSyncQueryService sqlSyncQueryService; + private EngineConfigSource engineConfigSource; + private static final String TRAFFIC_SPECTRUM_SUMMARY = "TRAFFIC_SPECTRUM_SUMMARY"; + private static final String TRAFFIC_SPECTRUM_UNIQ_IP = "TRAFFIC_SPECTRUM_UNIQ_IP"; + private static final String TRAFFIC_SPECTRUM_APP_DISTRIBUTION_SERVER_IP = "TRAFFIC_SPECTRUM_APP_DISTRIBUTION_SERVER_IP"; + private static final String TRAFFIC_SPECTRUM_APP_DISTRIBUTION_SERVER_DOMAIN = "TRAFFIC_SPECTRUM_APP_DISTRIBUTION_SERVER_DOMAIN"; + private static final String TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE = "TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE"; + private static final String TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND = "TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND"; + private static final String TABLE_TRAFFIC_SKETCH_METRIC = "traffic_sketch_metric"; + private static final String TIME_FIELD_TYPE = "unix_timestamp"; + private static final String GRANULARITY_DEFAULT = "PT60S"; + + @Override + public BaseResult run(DSLQueryRequestParam dslQueryRequestParam) { + if (JobConfig.TRAFFIC_SPECTRUM_SUMMARY.equals(dslQueryRequestParam.getName())) { + return BaseResultGenerator.success(summary(dslQueryRequestParam)); + } else if (JobConfig.TRAFFIC_SPECTRUM_UNIQUE_CLIENT_AND_SERVER_IPS.equals(dslQueryRequestParam.getName())) { + return BaseResultGenerator.success(uniqIpsStat(dslQueryRequestParam)); + } else if (JobConfig.TRAFFIC_SPECTRUM_APP_DISTRIBUTION.equals(dslQueryRequestParam.getName())) { + return BaseResultGenerator.success(topServerIpAndServerDomain(dslQueryRequestParam)); + } else if (JobConfig.TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE.equals(dslQueryRequestParam.getName())) { + return BaseResultGenerator.success(internalExternalDimensionMetricsStat(dslQueryRequestParam)); + } else if (JobConfig.TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND.equals(dslQueryRequestParam.getName())) { + return BaseResultGenerator.success(networkThroughputTrend(dslQueryRequestParam)); + } + throw new QGWBusinessException(HttpStatusCodeEnum.NOT_FOUND.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), dslQueryRequestParam.getName() + " was not found.")); + } + + @Override + public List<Map<String, Object>> summary(DSLQueryRequestParam dslQueryRequestParam) { + DSLQueryContext dslQueryContext = BeanUtil.copyProperties(dslQueryRequestParam, DSLQueryContext.class); + String sql = dslQueryContext.toSql( + environment.getProperty(TRAFFIC_SPECTRUM_SUMMARY) + , TABLE_TRAFFIC_SKETCH_METRIC + , databaseService.getPartitionKey(TABLE_TRAFFIC_SKETCH_METRIC) + , TIME_FIELD_TYPE); + BaseResult<List<Map<String, Object>>> baseResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder() + .originalSQL(sql) + .option(QueryOption.REAL_TIME.getValue()) + .build()); + if (baseResult.isSuccess()) { + baseResult.getData().forEach(map -> { + long bytes = Long.parseLong(String.valueOf(map.get("total_bytes") == null ? 0 : map.get("total_bytes"))); + long packets = Long.parseLong(String.valueOf(map.get("total_packets") == null ? 0 : map.get("total_packets"))); + long sessions = Long.parseLong(String.valueOf(map.get("total_sessions") == null ? 0 : map.get("total_sessions"))); + long unknownAppBytes = Long.parseLong(String.valueOf(map.get("unknown_app_bytes") == null ? 0 : map.get("unknown_app_bytes"))); + long asymmetricFlows = Long.parseLong(String.valueOf(map.get("asymmetric_flows") == null ? 0 : map.get("asymmetric_flows"))); + long fragmentationPackets = Long.parseLong(String.valueOf(map.get("fragmentation_packets") == null ? 0 : map.get("fragmentation_packets"))); + HashMap<String, Object> item = Maps.newLinkedHashMap(); + item.put(TrafficSpectrumConstants.DIRECTION, map.get("direction")); + item.put(TrafficSpectrumConstants.BYTES, bytes); + item.put(TrafficSpectrumConstants.PACKETS, packets); + item.put(TrafficSpectrumConstants.SESSIONS, sessions); + item.put(TrafficSpectrumConstants.UNKNOWN_APP_BYTES, unknownAppBytes); + item.put(TrafficSpectrumConstants.UNCATEGORIZED_PERCENT, bytes == 0 ? 0.0 : NumberUtil.div(BigDecimal.valueOf(unknownAppBytes), BigDecimal.valueOf(bytes), 4).doubleValue()); + item.put(TrafficSpectrumConstants.ASYMMETRIC_FLOWS, asymmetricFlows); + item.put(TrafficSpectrumConstants.ASYMMETRIC_FLOWS_PERCENT, sessions == 0 ? 0.0 : NumberUtil.div(BigDecimal.valueOf(asymmetricFlows), BigDecimal.valueOf(sessions), 4).doubleValue()); + item.put(TrafficSpectrumConstants.FRAGMENTATION_PACKETS, fragmentationPackets); + item.put(TrafficSpectrumConstants.FRAGMENTATION_PERCENT, packets == 0 ? 0.0 : NumberUtil.div(BigDecimal.valueOf(fragmentationPackets), BigDecimal.valueOf(packets), 4).doubleValue()); + map.clear(); + map.putAll(item); + }); + return baseResult.getData(); + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + } + + @Override + public List<Map<String, Object>> uniqIpsStat(DSLQueryRequestParam dslQueryRequestParam) { + DSLQueryContext dslQueryContext = BeanUtil.copyProperties(dslQueryRequestParam, DSLQueryContext.class); + String sql = dslQueryContext.toSql( + environment.getProperty(TRAFFIC_SPECTRUM_UNIQ_IP) + , TABLE_TRAFFIC_SKETCH_METRIC + , databaseService.getPartitionKey(TABLE_TRAFFIC_SKETCH_METRIC) + , TIME_FIELD_TYPE); + BaseResult<List<Map<String, Object>>> baseResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder() + .originalSQL(sql) + .option(QueryOption.REAL_TIME.getValue()) + .build()); + if (baseResult.isSuccess()) { + baseResult.getData().forEach(map -> { + long internalUniqClientIp = Long.parseLong(String.valueOf(map.get("internal_uniq_client_ip") == null ? 0 : map.get("internal_uniq_client_ip"))); + long internalUniqServerIp = Long.parseLong(String.valueOf(map.get("internal_uniq_server_ip") == null ? 0 : map.get("internal_uniq_server_ip"))); + long externalUniqClientIp = Long.parseLong(String.valueOf(map.get("external_uniq_client_ip") == null ? 0 : map.get("external_uniq_client_ip"))); + long externalUniqServerIp = Long.parseLong(String.valueOf(map.get("external_uniq_server_ip") == null ? 0 : map.get("external_uniq_server_ip"))); + HashMap<String, Object> item = Maps.newLinkedHashMap(); + item.put(TrafficSpectrumConstants.INTERNAL_UNIQ_CLIENT_IP, internalUniqClientIp); + item.put(TrafficSpectrumConstants.INTERNAL_UNIQ_SERVER_IP, internalUniqServerIp); + item.put(TrafficSpectrumConstants.EXTERNAL_UNIQ_CLIENT_IP, externalUniqClientIp); + item.put(TrafficSpectrumConstants.EXTERNAL_UNIQ_SERVER_IP, externalUniqServerIp); + map.clear(); + map.putAll(item); + }); + return baseResult.getData(); + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + } + + @Override + public List<Map<String, Object>> topServerIpAndServerDomain(DSLQueryRequestParam dslQueryRequestParam) { + DSLQueryContext dslQueryContext = BeanUtil.copyProperties(dslQueryRequestParam, DSLQueryContext.class); + String limit = dslQueryContext.getLimit(); + if (StrUtil.isBlank(limit)) { + dslQueryContext.setLimit("10"); + } + String sqlIp = dslQueryContext.toSql( + environment.getProperty(TRAFFIC_SPECTRUM_APP_DISTRIBUTION_SERVER_IP) + , TABLE_TRAFFIC_SKETCH_METRIC + , databaseService.getPartitionKey(TABLE_TRAFFIC_SKETCH_METRIC) + , TIME_FIELD_TYPE); + BaseResult<List<Map<String, Object>>> baseResultIp = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder() + .originalSQL(sqlIp) + .option(QueryOption.REAL_TIME.getValue()) + .build()); + String sqlDomain = dslQueryContext.toSql( + environment.getProperty(TRAFFIC_SPECTRUM_APP_DISTRIBUTION_SERVER_DOMAIN) + , TABLE_TRAFFIC_SKETCH_METRIC + , databaseService.getPartitionKey(TABLE_TRAFFIC_SKETCH_METRIC) + , TIME_FIELD_TYPE); + BaseResult<List<Map<String, Object>>> baseResultDomain = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder() + .originalSQL(sqlDomain) + .option(QueryOption.REAL_TIME.getValue()) + .build()); + if (baseResultIp.isSuccess() && baseResultDomain.isSuccess()) { + if ((baseResultIp.getData() == null || baseResultIp.getData().isEmpty()) && (baseResultDomain.getData() == null || baseResultDomain.getData().isEmpty())) { + return Lists.newArrayList(); + } + Map<String, Object> map = Maps.newHashMap(); + map.put(TrafficSpectrumConstants.TOP_SERVER_IPS, baseResultIp.getData()); + map.put(TrafficSpectrumConstants.TOP_SERVER_DOMAINS, baseResultDomain.getData()); + return Lists.newArrayList(map); + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResultIp.getMessage() + "; " + baseResultDomain.getMessage())); + } + + @Override + public List<Map<String, Object>> internalExternalDimensionMetricsStat(DSLQueryRequestParam dslQueryRequestParam) { + DSLQueryContext dslQueryContext = BeanUtil.copyProperties(dslQueryRequestParam, DSLQueryContext.class); + dslQueryContext.setLimit(String.valueOf(engineConfigSource.getTrafficSpectrumClientIPAppResultNum())); + if (StrUtil.isBlank(dslQueryContext.getGranularity())) { + dslQueryContext.setGranularity(GRANULARITY_DEFAULT); + } + String sql = dslQueryContext.toSql( + environment.getProperty(TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE) + , TABLE_TRAFFIC_SKETCH_METRIC + , databaseService.getPartitionKey(TABLE_TRAFFIC_SKETCH_METRIC) + , TIME_FIELD_TYPE); + BaseResult<List<Map<String, Object>>> baseResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder() + .originalSQL(sql) + .option(QueryOption.REAL_TIME.getValue()) + .build()); + if (baseResult.isSuccess()) { + if (baseResult.getData() == null || baseResult.getData().isEmpty()) { + return Lists.newArrayList(); + } + List<Map<String, Object>> data = baseResult.getData(); + Map<String, Object> nodes = Maps.newHashMap(); + Map<String, Map<String, Object>> internalNodes = Maps.newHashMap(); + Map<String, Map<String, Object>> externalNodes = Maps.newHashMap(); + List<Map<String, Object>> links = Lists.newArrayList(); + + data.forEach(x -> { + String direction = String.valueOf(x.get("direction")); + long sessions = Long.parseLong(String.valueOf(x.get("sessions") == null ? 0 : x.get("sessions"))); + long bytes = Long.parseLong(String.valueOf(x.get("bytes") == null ? 0 : x.get("bytes"))); + long packets = Long.parseLong(String.valueOf(x.get("packets") == null ? 0 : x.get("packets"))); + long maxRate = Long.parseLong(String.valueOf(x.get("max_rate") == null ? 0 : x.get("max_rate"))); + long avgRate = Long.parseLong(String.valueOf(x.get("avg_rate") == null ? 0 : x.get("avg_rate"))); + String clientIp = String.valueOf(x.get("client_ip")); + String app = String.valueOf(x.get("app")); + String clientCountry = String.valueOf(x.get("client_country")); + String appCategory = String.valueOf(x.get("app_category")); + + Map<String, Object> internalNode = Maps.newHashMap(); + Map<String, Object> externalNode = Maps.newHashMap(); + String uuidInternal; + String uuidExternal; + if (TrafficSpectrumConstants.OUTBOUND.equalsIgnoreCase(direction)) { + uuidInternal = UUID.nameUUIDFromBytes(StrUtil.concat(false, TrafficSpectrumConstants.INTERNAL_NODE, String.valueOf(clientIp), String.valueOf(clientCountry), TrafficSpectrumConstants.OBJECT_TYPE_IP).getBytes()).toString(); + internalNode.put(TrafficSpectrumConstants.ID, uuidInternal); + internalNode.put(TrafficSpectrumConstants.OBJECT_TYPE, TrafficSpectrumConstants.OBJECT_TYPE_IP); + internalNode.put(TrafficSpectrumConstants.NAME, clientIp); + internalNode.put(TrafficSpectrumConstants.CATEGORY, getIpCategory(clientIp, clientCountry)); + internalNodes.put(uuidInternal, internalNode); + + uuidExternal = UUID.nameUUIDFromBytes(StrUtil.concat(false, TrafficSpectrumConstants.EXTERNAL_NODE, String.valueOf(app), String.valueOf(appCategory), TrafficSpectrumConstants.OBJECT_TYPE_APPLICATION).getBytes()).toString(); + externalNode.put(TrafficSpectrumConstants.ID, uuidExternal); + externalNode.put(TrafficSpectrumConstants.OBJECT_TYPE, TrafficSpectrumConstants.OBJECT_TYPE_APPLICATION); + externalNode.put(TrafficSpectrumConstants.NAME, app); + externalNode.put(TrafficSpectrumConstants.CATEGORY, getApplicationCategory(appCategory)); + externalNodes.put(uuidExternal, externalNode); + + } else if (TrafficSpectrumConstants.INBOUND.equalsIgnoreCase(direction)) { + uuidExternal = UUID.nameUUIDFromBytes(StrUtil.concat(false, TrafficSpectrumConstants.EXTERNAL_NODE, String.valueOf(clientIp), String.valueOf(clientCountry), TrafficSpectrumConstants.OBJECT_TYPE_IP).getBytes()).toString(); + externalNode.put(TrafficSpectrumConstants.ID, uuidExternal); + externalNode.put(TrafficSpectrumConstants.OBJECT_TYPE, TrafficSpectrumConstants.OBJECT_TYPE_IP); + externalNode.put(TrafficSpectrumConstants.NAME, clientIp); + externalNode.put(TrafficSpectrumConstants.CATEGORY, getIpCategory(clientIp, clientCountry)); + externalNodes.put(uuidExternal, externalNode); + + uuidInternal = UUID.nameUUIDFromBytes(StrUtil.concat(false, TrafficSpectrumConstants.INTERNAL_NODE, String.valueOf(app), String.valueOf(appCategory), TrafficSpectrumConstants.OBJECT_TYPE_APPLICATION).getBytes()).toString(); + internalNode.put(TrafficSpectrumConstants.ID, uuidInternal); + internalNode.put(TrafficSpectrumConstants.OBJECT_TYPE, TrafficSpectrumConstants.OBJECT_TYPE_APPLICATION); + internalNode.put(TrafficSpectrumConstants.NAME, app); + internalNode.put(TrafficSpectrumConstants.CATEGORY, getApplicationCategory(appCategory)); + internalNodes.put(uuidInternal, internalNode); + } else { + log.warn("Unknown direction: {}", direction); + return; + } + Map<String, Object> link = Maps.newHashMap(); + link.put(TrafficSpectrumConstants.SOURCE, uuidInternal); + link.put(TrafficSpectrumConstants.TARGET, uuidExternal); + link.put(TrafficSpectrumConstants.DIRECTION, direction); + link.put(TrafficSpectrumConstants.SESSIONS, sessions); + link.put(TrafficSpectrumConstants.BYTES, bytes); + link.put(TrafficSpectrumConstants.PACKETS, packets); + link.put(TrafficSpectrumConstants.MAX_RATE, maxRate); + link.put(TrafficSpectrumConstants.AVG_RATE, avgRate); + links.add(link); + + }); + nodes.put(TrafficSpectrumConstants.INTERNAL_NODE, new ArrayList<>(internalNodes.values())); + nodes.put(TrafficSpectrumConstants.EXTERNAL_NODE, new ArrayList<>(externalNodes.values())); + Map<String, Object> result = Maps.newHashMap(); + result.put(TrafficSpectrumConstants.NODES, nodes); + result.put(TrafficSpectrumConstants.LINKS, links); + return Lists.newArrayList(result); + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + } + + @Override + public List<Map<String, Object>> networkThroughputTrend(DSLQueryRequestParam dslQueryRequestParam) { + DSLQueryContext dslQueryContext = BeanUtil.copyProperties(dslQueryRequestParam, DSLQueryContext.class); + if (StrUtil.isBlank(dslQueryContext.getGranularity())) { + dslQueryContext.setGranularity(GRANULARITY_DEFAULT); + } + String sql = dslQueryContext.toSql( + environment.getProperty(TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND) + , TABLE_TRAFFIC_SKETCH_METRIC + , databaseService.getPartitionKey(TABLE_TRAFFIC_SKETCH_METRIC) + , TIME_FIELD_TYPE); + BaseResult baseResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder() + .originalSQL(sql) + .option(QueryOption.REAL_TIME.getValue()) + .build()); + if (baseResult.isSuccess()) { + return (List<Map<String, Object>>) baseResult.getData(); + } + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + } + + @Override + public Map<String, Object> getInternalExternalBipartiteGraph(Map<String, Object> data, Integer clientIPAppResultNum) { + Map<String, List<Map<String, Object>>> originalNodes = (Map<String, List<Map<String, Object>>>) data.get(TrafficSpectrumConstants.NODES); + List<Map<String, Object>> originalLinks = (List<Map<String, Object>>) data.get(TrafficSpectrumConstants.LINKS); + List<Map<String, Object>> originalExternalNodeList = originalNodes.get(TrafficSpectrumConstants.EXTERNAL_NODE); + List<Map<String, Object>> originalInternalNodeList = originalNodes.get(TrafficSpectrumConstants.INTERNAL_NODE); + // TODO 自动降序排序基于 bytes + originalLinks.sort((o1, o2) -> { + long bytes1 = Long.parseLong(String.valueOf(o1.get(TrafficSpectrumConstants.BYTES))); + long bytes2 = Long.parseLong(String.valueOf(o2.get(TrafficSpectrumConstants.BYTES))); + return Long.compare(bytes2, bytes1); + }); + // TODO step1 原数据拆封 nodes、links + // TODO 从 internalNodeList 和 externalNodeList 过滤除 name 不为Other的元素列表 和是Other的元素列表 + Map<String, Map<String, Object>> originalInternalOtherMap = Maps.newHashMap(); + Map<String, Map<String, Object>> originalInternalNoOtherMap = Maps.newHashMap(); + for (Map<String, Object> value : originalInternalNodeList) { + if (StrUtil.equals(String.valueOf(value.get(TrafficSpectrumConstants.NAME)), TrafficSpectrumConstants.OTHER)) { + originalInternalOtherMap.put(String.valueOf(value.get(TrafficSpectrumConstants.ID)), value); + } else { + originalInternalNoOtherMap.put(String.valueOf(value.get(TrafficSpectrumConstants.ID)), value); + } + } + + Map<String, Map<String, Object>> originalExternalOtherMap = Maps.newHashMap(); + Map<String, Map<String, Object>> originalExternalNoOtherMap = Maps.newHashMap(); + for (Map<String, Object> value : originalExternalNodeList) { + if (StrUtil.equals(String.valueOf(value.get(TrafficSpectrumConstants.NAME)), TrafficSpectrumConstants.OTHER)) { + originalExternalOtherMap.put(String.valueOf(value.get(TrafficSpectrumConstants.ID)), value); + } else { + originalExternalNoOtherMap.put(String.valueOf(value.get(TrafficSpectrumConstants.ID)), value); + } + } + + List<Map<String, Object>> originalOtherLinks = new ArrayList<>(); + List<Map<String, Object>> originalNoOtherLinks = new ArrayList<>(); + for (Map<String, Object> value : originalLinks) { + String source = String.valueOf(value.get(TrafficSpectrumConstants.SOURCE)); + String target = String.valueOf(value.get(TrafficSpectrumConstants.TARGET)); + if (originalExternalOtherMap.containsKey(source) + || originalExternalOtherMap.containsKey(target) + || originalInternalOtherMap.containsKey(source) + || originalInternalOtherMap.containsKey(target)) { + originalOtherLinks.add(value); + } else { + originalNoOtherLinks.add(value); + } + } + + if (clientIPAppResultNum == null || clientIPAppResultNum >= originalNoOtherLinks.size()) { + return data; + } + + // TODO step2 结果封装 + // TODO 生成结果 nods + Map<String, Map<String, Object>> resultNoOtherInternalNodeMap = Maps.newHashMap(); + Map<String, Map<String, Object>> resultNoOtherExternalNodeMap = Maps.newHashMap(); + Set<String> sourceSets = Sets.newHashSet(); + Set<String> targetSets = Sets.newHashSet(); + for (Map<String, Object> map : originalNoOtherLinks) { + String source = String.valueOf(map.get(TrafficSpectrumConstants.SOURCE)); + sourceSets.add(source); + String target = String.valueOf(map.get(TrafficSpectrumConstants.TARGET)); + targetSets.add(target); + if (sourceSets.size() > clientIPAppResultNum || targetSets.size() > clientIPAppResultNum) { + continue; + } + resultNoOtherInternalNodeMap.put(source, originalInternalNoOtherMap.get(source)); + resultNoOtherExternalNodeMap.put(target, originalExternalNoOtherMap.get(target)); + } + if (sourceSets.size() <= clientIPAppResultNum || targetSets.size() <= clientIPAppResultNum) { + return data; + } + + Map<String, Map<String, Object>> resultOtherInternalNodeMap = Maps.newHashMap(); + Map<String, Map<String, Object>> resultOtherExternalNodeMap = Maps.newHashMap(); + for (Map<String, Object> originalItem : originalInternalNodeList) { + String originalId = String.valueOf(originalItem.get(TrafficSpectrumConstants.ID)); + if (resultNoOtherInternalNodeMap.containsKey(originalId)) { + continue; + } + Map<String, Object> otherInternalNodeMapItem = Maps.newHashMap(); + String category = String.valueOf(originalItem.get(TrafficSpectrumConstants.CATEGORY)); + String objectType = String.valueOf(originalItem.get(TrafficSpectrumConstants.OBJECT_TYPE)); + otherInternalNodeMapItem.put(TrafficSpectrumConstants.OBJECT_TYPE, objectType); + boolean presentCategory = resultNoOtherInternalNodeMap.values().stream().anyMatch(o -> objectType.equals(o.get(TrafficSpectrumConstants.OBJECT_TYPE)) && category.equals(o.get(TrafficSpectrumConstants.CATEGORY))); + if (presentCategory) { + otherInternalNodeMapItem.put(TrafficSpectrumConstants.CATEGORY, category); + } else { + otherInternalNodeMapItem.put(TrafficSpectrumConstants.CATEGORY, TrafficSpectrumConstants.OTHER); + } + otherInternalNodeMapItem.put(TrafficSpectrumConstants.NAME, TrafficSpectrumConstants.OTHER); + String unoriginalId = UUID.nameUUIDFromBytes(StrUtil.concat(false, TrafficSpectrumConstants.INTERNAL_NODE, String.valueOf(otherInternalNodeMapItem.get(TrafficSpectrumConstants.NAME)), String.valueOf(otherInternalNodeMapItem.get(TrafficSpectrumConstants.CATEGORY)), String.valueOf(otherInternalNodeMapItem.get(TrafficSpectrumConstants.OBJECT_TYPE))).getBytes()).toString(); + otherInternalNodeMapItem.put(TrafficSpectrumConstants.ID, unoriginalId); + resultOtherInternalNodeMap.put(originalId, otherInternalNodeMapItem); + } + + for (Map<String, Object> originalItem : originalExternalNodeList) { + String originalId = String.valueOf(originalItem.get(TrafficSpectrumConstants.ID)); + if (resultNoOtherExternalNodeMap.containsKey(originalId)) { + continue; + } + Map<String, Object> otherExternalNodeMapItem = Maps.newHashMap(); + String category = String.valueOf(originalItem.get(TrafficSpectrumConstants.CATEGORY)); + String objectType = String.valueOf(originalItem.get(TrafficSpectrumConstants.OBJECT_TYPE)); + otherExternalNodeMapItem.put(TrafficSpectrumConstants.OBJECT_TYPE, objectType); + boolean presentCategory = resultNoOtherExternalNodeMap.values().stream().anyMatch(o -> objectType.equals(o.get(TrafficSpectrumConstants.OBJECT_TYPE)) && category.equals(o.get(TrafficSpectrumConstants.CATEGORY))); + if (!presentCategory) { + otherExternalNodeMapItem.put(TrafficSpectrumConstants.CATEGORY, TrafficSpectrumConstants.OTHER); + } else { + otherExternalNodeMapItem.put(TrafficSpectrumConstants.CATEGORY, category); + } + otherExternalNodeMapItem.put(TrafficSpectrumConstants.NAME, TrafficSpectrumConstants.OTHER); + String unoriginalId = UUID.nameUUIDFromBytes(StrUtil.concat(false, TrafficSpectrumConstants.EXTERNAL_NODE, String.valueOf(otherExternalNodeMapItem.get(TrafficSpectrumConstants.NAME)), String.valueOf(otherExternalNodeMapItem.get(TrafficSpectrumConstants.CATEGORY)), String.valueOf(otherExternalNodeMapItem.get(TrafficSpectrumConstants.OBJECT_TYPE))).getBytes()).toString(); + otherExternalNodeMapItem.put(TrafficSpectrumConstants.ID, unoriginalId); + resultOtherExternalNodeMap.put(originalId, otherExternalNodeMapItem); + } + + // TODO 基于 originalLinks 生成结果 resultLinks + List<Map<String, Object>> resultLinks = Lists.newArrayList(originalLinks); + for (Map<String, Object> resultLink : resultLinks) { + String source = String.valueOf(resultLink.get(TrafficSpectrumConstants.SOURCE)); + String target = String.valueOf(resultLink.get(TrafficSpectrumConstants.TARGET)); + if (resultOtherInternalNodeMap.containsKey(source)) { + resultLink.put(TrafficSpectrumConstants.SOURCE, resultOtherInternalNodeMap.get(source).get(TrafficSpectrumConstants.ID)); + } + if (resultOtherExternalNodeMap.containsKey(target)) { + resultLink.put(TrafficSpectrumConstants.TARGET, resultOtherExternalNodeMap.get(target).get(TrafficSpectrumConstants.ID)); + } + } + + Map<String, Object> result = Maps.newHashMap(); + List<Map<String, Object>> links = mergeLinks(resultLinks); + Map<String, List<Map<String, Object>>> nodes = Maps.newHashMap(); + List<Map<String, Object>> internalNode = Lists.newArrayList(); + List<Map<String, Object>> externalNode = Lists.newArrayList(); + internalNode.addAll(resultOtherInternalNodeMap.values()); + internalNode.addAll(resultNoOtherInternalNodeMap.values()); + externalNode.addAll(resultOtherExternalNodeMap.values()); + externalNode.addAll(resultNoOtherExternalNodeMap.values()); + nodes.put(TrafficSpectrumConstants.INTERNAL_NODE, internalNode); + nodes.put(TrafficSpectrumConstants.EXTERNAL_NODE, externalNode); + Map<String, List<Map<String, Object>>> distinctNodes = distinctNodes(nodes); + result.put(TrafficSpectrumConstants.NODES, distinctNodes); + result.put(TrafficSpectrumConstants.LINKS, links); + return result; + } + + @Override + public Map<String, Object> mergeInternalExternalBipartiteGraph(Map<String, Object> data1, Map<String, Object> data2) { + Map<String, List<Map<String, Object>>> nodes1 = (Map<String, List<Map<String, Object>>>) data1.get(TrafficSpectrumConstants.NODES); + List<Map<String, Object>> links1 = (List<Map<String, Object>>) data1.get(TrafficSpectrumConstants.LINKS); + List<Map<String, Object>> externalNodeList1 = nodes1.get(TrafficSpectrumConstants.EXTERNAL_NODE); + List<Map<String, Object>> internalNodeList1 = nodes1.get(TrafficSpectrumConstants.INTERNAL_NODE); + + Map<String, List<Map<String, Object>>> nodes2 = (Map<String, List<Map<String, Object>>>) data2.get(TrafficSpectrumConstants.NODES); + List<Map<String, Object>> links2 = (List<Map<String, Object>>) data2.get(TrafficSpectrumConstants.LINKS); + List<Map<String, Object>> externalNodeList2 = nodes2.get(TrafficSpectrumConstants.EXTERNAL_NODE); + List<Map<String, Object>> internalNodeList2 = nodes2.get(TrafficSpectrumConstants.INTERNAL_NODE); + // TODO 合并 data1、data2，并且去重 + List<Map<String, Object>> lists = Lists.newArrayList(); + lists.addAll(links1); + lists.addAll(links2); + List<Map<String, Object>> distinctLinks = mergeLinks(lists); + + Map<String, List<Map<String, Object>>> nodes = Maps.newHashMap(); + List<Map<String, Object>> externalNode = Lists.newArrayList(); + externalNode.addAll(externalNodeList1); + externalNode.addAll(externalNodeList2); + List<Map<String, Object>> internalNode = Lists.newArrayList(); + internalNode.addAll(internalNodeList1); + internalNode.addAll(internalNodeList2); + nodes.put(TrafficSpectrumConstants.INTERNAL_NODE, internalNode); + nodes.put(TrafficSpectrumConstants.EXTERNAL_NODE, externalNode); + Map<String, List<Map<String, Object>>> distinctNodes = distinctNodes(nodes); + Map<String, Object> data = Maps.newHashMap(); + data.put(TrafficSpectrumConstants.NODES, distinctNodes); + data.put(TrafficSpectrumConstants.LINKS, distinctLinks); + int sqrt = (int) Math.sqrt(engineConfigSource.getMaxCacheNum()); + return getInternalExternalBipartiteGraph(data, Math.max(sqrt, engineConfigSource.getTrafficSpectrumClientIPAppResultNum())); + } + + @Override + public List<Map<String, Object>> mergeLinks(List<Map<String, Object>> links) { + Map<String, Map<String, Object>> resultLinksMap = Maps.newHashMap(); + links.forEach(o -> { + String uniqueKey = DigestUtil.md5Hex(o.get(TrafficSpectrumConstants.SOURCE) + + TrafficSpectrumConstants.SPLIT_HYPHEN + o.get(TrafficSpectrumConstants.DIRECTION) + + TrafficSpectrumConstants.SPLIT_HYPHEN + o.get(TrafficSpectrumConstants.TARGET)); + if (resultLinksMap.containsKey(uniqueKey)) { + Map<String, Object> map = resultLinksMap.get(uniqueKey); + map.put(TrafficSpectrumConstants.SESSIONS, Long.parseLong(String.valueOf(map.get(TrafficSpectrumConstants.SESSIONS))) + Long.parseLong(String.valueOf(o.get(TrafficSpectrumConstants.SESSIONS)))); + Long bytes1 = Long.parseLong(String.valueOf(map.get(TrafficSpectrumConstants.BYTES))); + Long bytes2 = Long.parseLong(String.valueOf(o.get(TrafficSpectrumConstants.BYTES))); + map.put(TrafficSpectrumConstants.BYTES, bytes1 + bytes2); + map.put(TrafficSpectrumConstants.PACKETS, Long.parseLong(String.valueOf(map.get(TrafficSpectrumConstants.PACKETS))) + Long.parseLong(String.valueOf(o.get(TrafficSpectrumConstants.PACKETS)))); + map.put(TrafficSpectrumConstants.MAX_RATE, Math.max(Long.parseLong(String.valueOf(map.get(TrafficSpectrumConstants.MAX_RATE))), Long.parseLong(String.valueOf(o.get(TrafficSpectrumConstants.MAX_RATE))))); + + //TODO 计算 avg_rate 时，通过 bytes 权重，计算加权平均值 + Long avgRate1 = Long.parseLong(String.valueOf(map.get(TrafficSpectrumConstants.AVG_RATE))); + Long avgRate2 = Long.parseLong(String.valueOf(o.get(TrafficSpectrumConstants.AVG_RATE))); + if (bytes1 + bytes2 == 0) { + map.put(TrafficSpectrumConstants.AVG_RATE, (avgRate1 + avgRate2) / 2); + } else { + long avg = (long) (avgRate1 * (bytes1 * 1.0 / (bytes1 + bytes2)) + avgRate2 * (bytes2 * 1.0 / (bytes1 + bytes2))); + map.put(TrafficSpectrumConstants.AVG_RATE, avg); + } + resultLinksMap.put(uniqueKey, map); + } else { + resultLinksMap.put(uniqueKey, o); + } + }); + List<Map<String, Object>> result = new ArrayList<>(resultLinksMap.values()); + result.sort((o1, o2) -> { + long bytes1 = Long.parseLong(String.valueOf(o1.get(TrafficSpectrumConstants.BYTES))); + long bytes2 = Long.parseLong(String.valueOf(o2.get(TrafficSpectrumConstants.BYTES))); + return Long.compare(bytes2, bytes1); + }); + return result; + } + + @Override + public Map<String, List<Map<String, Object>>> distinctNodes(Map<String, List<Map<String, Object>>> nodes) { + Map<String, List<Map<String, Object>>> resultNode = Maps.newHashMap(); + List<Map<String, Object>> internalNodes = nodes.get(TrafficSpectrumConstants.INTERNAL_NODE); + List<Map<String, Object>> externalNodes = nodes.get(TrafficSpectrumConstants.EXTERNAL_NODE); + List<Map<String, Object>> resultInternalNodes = internalNodes.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(o -> o.get(TrafficSpectrumConstants.ID).toString()))), ArrayList::new)); + List<Map<String, Object>> resultExternalNodes = externalNodes.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(o -> o.get(TrafficSpectrumConstants.ID).toString()))), ArrayList::new)); + resultNode.put(TrafficSpectrumConstants.INTERNAL_NODE, resultInternalNodes); + resultNode.put(TrafficSpectrumConstants.EXTERNAL_NODE, resultExternalNodes); + return resultNode; + } + + private String getApplicationCategory(String appCategory) { + if (StrUtil.isEmpty(appCategory)) { + return TrafficSpectrumConstants.OTHER; + } + return appCategory; + } + + private String getIpCategory(String clientIp, String clientCountry) { + if (!StrUtil.isEmpty(clientCountry)) { + return clientCountry; + } + if (StrUtil.isEmpty(clientCountry) && IPUtil.isIPv4Address(clientIp) && IPUtil.internalIp(clientIp)) { + return TrafficSpectrumConstants.PRIVATE_IP; + } + return TrafficSpectrumConstants.OTHER; + } + + @Autowired + public void setEnvironment(Environment environment) { + this.environment = environment; + } + + @Autowired + public void setDatabaseService(DatabaseService databaseService) { + this.databaseService = databaseService; + } + + @Autowired + public void setSqlSyncQueryService(SQLSyncQueryService sqlSyncQueryService) { + this.sqlSyncQueryService = sqlSyncQueryService; + } + + @Autowired + public void setEngineConfigSource(EngineConfigSource engineConfigSource) { + this.engineConfigSource = engineConfigSource; + } +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/TroubleshootingServiceImp.java b/src/main/java/com/mesalab/qgw/service/impl/TroubleshootingServiceImp.java new file mode 100644 index 00000000..3b74a3af --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/TroubleshootingServiceImp.java @@ -0,0 +1,789 @@ +package com.mesalab.qgw.service.impl; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.net.NetUtil; +import cn.hutool.core.util.IdUtil; +import cn.hutool.core.util.NumberUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.core.util.URLUtil; +import cn.hutool.json.JSONUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; +import com.alibaba.fastjson2.JSONPath; +import com.alibaba.nacos.api.config.ConfigService; +import com.alibaba.nacos.api.exception.NacosException; +import com.geedgenetworks.utils.CommonUtil; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.StringUtil; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; +import com.jayway.jsonpath.JsonPath; +import com.jfinal.plugin.activerecord.Db; +import com.jfinal.plugin.activerecord.Record; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.entity.DataTypeMapping; +import com.mesalab.common.enums.*; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.nacos.NacosConfig; +import com.mesalab.common.utils.JsonSchemaValidator; +import com.mesalab.qgw.benchmark.DialectWriter; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.*; +import com.mesalab.qgw.service.*; +import com.opencsv.CSVWriter; +import org.apache.commons.math3.stat.StatUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.EnvironmentAware; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Service("troubleshootingService") +public class TroubleshootingServiceImp implements TroubleshootingService, EnvironmentAware { + + private static final Log log = LogFactory.get(); + private static final String BACKEND_ENGINE = "backend_engine"; + private static final String IDENTIFIER_NAME = "identifier_name"; + private static final String TEMPLATE = "template"; + private static final String TYPE = "type"; + + private static final String DATASET_TYPE = "type"; + + @Autowired + private DatabaseService databaseService; + @Autowired + private ClickHouseHttpSource clickHouseHttpSource; + @Autowired + private DruidIoHttpSource druidIoHttpSource; + @Autowired + private HBaseAPISource hBaseAPISource; + + @Autowired + private SQLSyncQueryService sqlSyncQueryService; + @Autowired + private HttpClientService httpClientService; + @Autowired + private HttpClientServiceV2 httpClientServiceV2; + @Autowired + HttpConfig httpConfig; + @Autowired + private DatasetService datasetService; + @Autowired + private DialectWriter dialectWriter; + @Autowired + private ConfigService systemConfigService; + @Autowired + private NacosConfig nacosConfig; + @Autowired + private QueryJobService queryJobService; + @Value("${server.port}") + private int serverPort; + @Value("${management.endpoints.web.base-path}") + private String basePath; + @Value("${management.endpoints.web.path-mapping.health}") + private String healthPath; + + private Environment env; + private static Pattern pTTL = Pattern.compile(".*toIntervalSecond\\((\\d+)\\)", Pattern.CASE_INSENSITIVE); + private static final String trafficPort = "8123"; + private final static String DATA_ID = "version.json"; + private final static String PUBLIC_SCHEMA_INFO = "public_schema_info.json"; + private static final String FIELDS = "fields"; + private static final String NAMESPACE = "namespace"; + private static final String SCHEMA_SYNTAX_RULE = "schema-syntax-validation.json"; + + private static String localHostAddress; + private final static Map<String, String> metaMap = Maps.newHashMap(); + + static { + localHostAddress = NetUtil.getLocalhostStr(); + } + + static { + metaMap.put("Int8", DataTypeMapping.INT); + metaMap.put("Int16", DataTypeMapping.INT); + metaMap.put("Int32", DataTypeMapping.INT); + metaMap.put("Int64", DataTypeMapping.LONG); + metaMap.put("Int128", DataTypeMapping.LONG); + metaMap.put("Int256", DataTypeMapping.LONG); + metaMap.put("UInt8", DataTypeMapping.INT); + metaMap.put("UInt16", DataTypeMapping.INT); + metaMap.put("UInt32", DataTypeMapping.LONG); + metaMap.put("UInt64", DataTypeMapping.LONG); + metaMap.put("UInt256", DataTypeMapping.LONG); + metaMap.put("Nullable(Int32)", DataTypeMapping.INT); + metaMap.put("Nullable(Int64)", DataTypeMapping.LONG); + metaMap.put("Nullable(Float64)", DataTypeMapping.DOUBLE); + metaMap.put("BIGINT", DataTypeMapping.LONG); + metaMap.put("Float32", DataTypeMapping.FLOAT); + metaMap.put("Float64", DataTypeMapping.DOUBLE); + metaMap.put("Date", DataTypeMapping.DATE); + metaMap.put("DateTime", DataTypeMapping.TIMESTAMP); + metaMap.put("DateTime64(3)", DataTypeMapping.STRING); + metaMap.put("TIMESTAMP", DataTypeMapping.STRING); + metaMap.put("Array(Int64)", DataTypeMapping.ARRAY.concat("(").concat(DataTypeMapping.LONG).concat(")")); + metaMap.put("Array(Int32)", DataTypeMapping.ARRAY.concat("(").concat(DataTypeMapping.INT).concat(")")); + metaMap.put("Array(String)", DataTypeMapping.ARRAY.concat("(").concat(DataTypeMapping.STRING).concat(")")); + metaMap.put("VARCHAR", DataTypeMapping.STRING); + metaMap.put("LowCardinality(String)", DataTypeMapping.STRING); + } + + @Override + public JSONObject getComponentStatus() { + Map<String, Object> jsonMap = Maps.newLinkedHashMap(); + try { + String content = systemConfigService.getConfig(DATA_ID, nacosConfig.getGroup(), 3000); + Map componentInfo = JSON.parseObject(content, Map.class); + List<Map<String, Object>> componentList = (List<Map<String, Object>>) componentInfo.get("components"); + Map<String, Object> componentHealthResult = getComponentHealth(); + Set<String> componentNameSet = componentHealthResult.keySet(); + boolean isStatusUp = true; + for (Map component : componentList) { + String componentName = String.valueOf(component.get("name")); + if (componentNameSet.contains(componentName)) { + Map<String, String> healthMap = buildHealth(componentName, componentHealthResult); + if (healthMap.get("status").equals("DOWN")) { + isStatusUp = false; + } + if (StringUtil.isNotEmpty(healthMap.get("version"))) { + component.put("version", healthMap.get("version")); + healthMap.remove("version"); + } + component.put("health", healthMap); + } + } + jsonMap.put("product", componentInfo.get("product")); + jsonMap.put("status", isStatusUp ? "UP" : "DOWN"); + jsonMap.put("components", componentList); + jsonMap.put("registered", componentInfo.get("registered")); + jsonMap.put("version", componentInfo.get("version")); + jsonMap.put("updated", componentInfo.get("updated")); + } catch (NacosException e) { + e.printStackTrace(); + } + return new JSONObject(jsonMap); + } + + @Override + public BaseResult consistencyCheck() { + List<Map<String, Object>> dataList = Lists.newArrayList(); + Map<String, Object> logDiff = Maps.newHashMap(); + Map tables = databaseService.getSchemaInfo(MetadataType.TABLES.getValue(), clickHouseHttpSource.getDbName(), false); + List<String> symbols = (List<String>) tables.get("symbols"); + logDiff.put("logType", DatabaseServiceImpl.LogType.TRAFFIC_LOGS.getValue()); + Map<String, Map<String, Object>> changeTTL = Maps.newHashMap(); + List<String> ipPorts = getClusterAddressOfCK(); + try { + for (String ipPort : ipPorts) { + if (!changeTTL.isEmpty()) { + break; + } + if (ipPort.startsWith("127.0.0.1")) { + String url = clickHouseHttpSource.getUrl(); + ipPort = url.substring(url.indexOf("//") + 2, url.lastIndexOf(":") + 1).concat(trafficPort); + } + for (String tableName : symbols) { + Map schemaMap = databaseService.getSchemaInfo(FIELDS, tableName, false); + Object tableTTLInSchema = getTableTTLInSchema(schemaMap); + Map<String, Object> tableDiff = getTableDiff(ipPort, tableName, tableTTLInSchema); + List fieldDiff = getFieldDiff(ipPort, tableName, schemaMap); + if (CollectionUtil.isNotEmpty(fieldDiff)) { + tableDiff.put(FIELDS, fieldDiff); + + } + if (CollectionUtil.isNotEmpty(tableDiff)) { + changeTTL.put(tableName, tableDiff); + logDiff.put("tables", Lists.newArrayList(changeTTL)); + logDiff.put("address", Lists.newArrayList(ipPort)); + } + } + } + } catch (RuntimeException ex) { + log.error(" schema TTL check error: {}", ex); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), ex.getMessage())); + } + if (changeTTL.isEmpty()) { + return BaseResultGenerator.success("ok", dataList); + } + dataList.add(logDiff); + return BaseResultGenerator.success("Log streaming setting task not executed or failed", dataList); + } + + @Override + public BaseResult validateMetadata() { + BaseResult result = null; + try { + Stopwatch watch = Stopwatch.createStarted(); + Map<String, Object> statistics = Maps.newLinkedHashMap(); + Map<String, Map<String, List<String>>> resultData = Maps.newHashMap(); + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + for (String tableName : databaseService.getAllTable()) { + Map schemaMap = databaseService.getSchemaInfo(FIELDS, tableName, false); + if (!clickHouseHttpSource.getDbName().equals(schemaMap.get(NAMESPACE)) && !druidIoHttpSource.getDbname().equals(schemaMap.get(NAMESPACE))) + continue; + jsonSchemaValidator + .addRule(SCHEMA_SYNTAX_RULE, "schema") + .validateSchema(JSON.toJSONString(schemaMap)); + } + checkCKIndexKey(clickHouseHttpSource.getDbName(), databaseService.getAllTable()); + checkMetadataFieldAndType(databaseService.getAllTable(), resultData); + checkMetadataFunctionConstraint(databaseService.getAllTable()); + + statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); + result = BaseResultGenerator.success("Json Schema Validation Success.", resultData, statistics); + result.setOutputMode(OutputMode.JSON.getValue()); + } catch (RuntimeException e) { + log.error("Json Schema Validation Fail: {}", e.getMessage()); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), e.getMessage())); + } + return result; + } + + + @Override + public BaseResult datesetVerification() { + List<Record> datasetList = Db.find(String.format(Objects.requireNonNull(env.getProperty("SQL_DATASETS")), "")); + List<Map<String, String>> sqlDatasetList = Lists.newArrayList(); + List<Map<String, String>> dslDatasetList = Lists.newArrayList(); + List<Map<String, String>> resultData = Lists.newArrayList(); + List<LinkedHashMap> variables = datasetService.getVariable(); + datasetList.forEach(record -> { + Map<String, String> map = Maps.newHashMap(); + map.put(IDENTIFIER_NAME, String.valueOf(record.getColumns().get(IDENTIFIER_NAME))); + map.put(TEMPLATE, String.valueOf(datasetService.buildExecSQL(variables, String.valueOf(record.getColumns().get(TEMPLATE))))); + if (record.getColumns().get(DATASET_TYPE).equals("sql")) { + sqlDatasetList.add(map); + } else if (record.getColumns().get(DATASET_TYPE).equals("dsl")) { + dslDatasetList.add(map); + } + }); + + SqlQueryRequestParam sqlQueryRequest; + for (Map sqlDataset : sqlDatasetList) { + Map<String, String> map = Maps.newHashMap(); + try { + sqlQueryRequest = JSON.parseObject(String.valueOf(sqlDataset.get(TEMPLATE)), SqlQueryRequestParam.class); + sqlQueryRequest.setExecutionMode(ExecutionMode.ONESHOT.getValue()); + sqlQueryRequest.setIsDryRun(1); + sqlQueryRequest.setOutputMode(OutputMode.JSON.getValue()); + log.info("SQL Dataset Verification, ID is: {}", sqlDataset.get(IDENTIFIER_NAME)); + queryJobService.createSQLAdHocQuery(sqlQueryRequest); + } catch (QGWBusinessException | BusinessException e) { + map.put(String.valueOf(sqlDataset.get(IDENTIFIER_NAME)), e.getMessage()); + resultData.add(map); + log.error("SQL Dataset execute error, ID is: {}, message is: {}", sqlDataset.get(IDENTIFIER_NAME), e.getMessage()); + } + } + + DSLQueryRequestParam dslQueryRequest; + for (Map dslDataset : dslDatasetList) { + Map<String, String> map = Maps.newHashMap(); + try { + dslQueryRequest = JSON.parseObject(String.valueOf(dslDataset.get(TEMPLATE)), DSLQueryRequestParam.class); + dslQueryRequest.setExecutionMode(ExecutionMode.ONESHOT.getValue()); + dslQueryRequest.setIsDryRun(1); + dslQueryRequest.setOutputMode(OutputMode.JSON.getValue()); + log.info("DSL Dataset Verification, ID is {}", dslDataset.get(IDENTIFIER_NAME)); + queryJobService.createDSLAdHocQuery(dslQueryRequest); + } catch (RuntimeException e) { + map.put(String.valueOf(dslDataset.get(IDENTIFIER_NAME)), e.getMessage()); + resultData.add(map); + log.error("DSL Dataset execute error, ID is: {}, message is: {}", dslDataset.get(IDENTIFIER_NAME), e.getMessage()); + } + } + + return buildResult(variables ,resultData, sqlDatasetList.size() + dslDatasetList.size()); + } + + @Override + public BaseResult benchmarkTest(String test, boolean isSaved) { + + BaseResult result = null; + Map<String, Object> dataResult = Maps.newLinkedHashMap(); + Map<String, Object> statistics = Maps.newLinkedHashMap(); + Map<String, Map<String, String>> mergeResult; + Map<String, Map<String, String>> engineResult; + Map<String, Map<String, String>> clickhouseResult; + Map<String, Map<String, String>> druidResult; + Map<String, Map<String, String>> hbaseResult; + try { + List<Map<String, Object>> dataSetlist = datasetService.getDatasets(Lists.newArrayList(), test, null); + List<Map> druidDatasetList = Lists.newLinkedList(); + List<Map> ckDatasetList = Lists.newLinkedList(); + List<Map> engineDatasetList = Lists.newLinkedList(); + List<Map> hbaseDatasetList = Lists.newLinkedList(); + List<LinkedHashMap> variables = datasetService.getVariable(); + + Stopwatch watch = Stopwatch.createStarted(); + if (CollectionUtil.isNotEmpty(dataSetlist)) { + for (int i = 0; i < dataSetlist.size(); i++) { + String backendEngine = String.valueOf(dataSetlist.get(i).get(BACKEND_ENGINE)); + String identifierName = String.valueOf(dataSetlist.get(i).get(IDENTIFIER_NAME)); + String template = String.valueOf(dataSetlist.get(i).get(TEMPLATE)); + String type = String.valueOf(dataSetlist.get(i).get(TYPE)); + Map<String, String> map = Maps.newHashMap(); + map.put(IDENTIFIER_NAME, identifierName); + map.put(TYPE, type); + map.put(TEMPLATE, String.valueOf(datasetService.buildExecSQL(variables, template))); + if (DBEngineType.DRUID.getValue().equalsIgnoreCase(backendEngine)) { + druidDatasetList.add(map); + } else if (DBEngineType.CLICKHOUSE.getValue().equalsIgnoreCase(backendEngine)) { + ckDatasetList.add(map); + } else if (DBEngineType.HBASE.getValue().equalsIgnoreCase(backendEngine)) { + hbaseDatasetList.add(map); + } else { + engineDatasetList.add(map); + } + } + } else { + return BaseResultGenerator.failure(HttpStatusCodeEnum.BAD_REQUEST.getCode(), "The dataset category does not exist."); + } + + clickhouseResult = buildResult(test, DBEngineType.CLICKHOUSE.getValue(), ckDatasetList, isSaved); + druidResult = buildResult(test, DBEngineType.DRUID.getValue(), druidDatasetList, isSaved); + engineResult = buildResult(test, DBEngineType.QGW.getValue(), engineDatasetList, isSaved); + hbaseResult = buildResult(test, DBEngineType.HBASE.getValue(), hbaseDatasetList, isSaved); + + mergeResult = Stream + .concat(clickhouseResult.entrySet().stream(), druidResult.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + mergeResult = Stream.concat(mergeResult.entrySet().stream(), engineResult.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + mergeResult = Stream.concat(mergeResult.entrySet().stream(), hbaseResult.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + dataResult.put(test, mergeResult); + statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); + result = BaseResultGenerator.success(StrUtil.format("SQL benchmark complete, time range is: {}", getTimeRangeByVariable(variables)), dataResult, statistics); + result.setOutputMode(OutputMode.JSON.getValue()); + } catch (RuntimeException e) { + log.error("Execute Poc SQL Fail:{}", e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), + CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(), e.getMessage())); + } + return result; + } + + private Map<String, Object> getComponentHealth() { + String healthCheckUrl = URLUtil.normalize(localHostAddress + ":" + serverPort + basePath + healthPath); + HttpResponseResult result = httpClientServiceV2.get(healthCheckUrl, httpConfig.getServerRequestTimeOut()); + Map<String, Object> componentsHealthResult = JSON.parseObject(result.getResponseBody(), Map.class); + Map<String, Object> componentMap = (Map<String, Object>) componentsHealthResult.get("components"); + return componentMap; + } + + + private Map<String, String> buildHealth(String componentName, Map<String, Object> map) { + Map<String, Object> componentMap = (Map<String, Object>) map.get(componentName); + Map<String, String> healthMap = (Map<String, String>) componentMap.get("details"); + healthMap.put("status", String.valueOf(componentMap.get("status"))); + healthMap.remove("app"); + return healthMap; + } + + private void checkCKIndexKey(String dbName, List<String> tables) { + for (String tableName : tables) { + LinkedHashMap<String, Object> schemaMap = databaseService.getSchemaByName(tableName); + if (!dbName.equals(schemaMap.get("namespace"))) continue; + BaseResult indexKeyResult = getBaseResultByEngine(SQLQueryContext.builder().option(QueryOption.REAL_TIME.getValue()).originalSQL(String.format(Objects.requireNonNull(env.getProperty("TABLE_INDEX_KEY")), tableName, clickHouseHttpSource.getDbName())).build()); + if (HttpStatusCodeEnum.SUCCESS.getCode() != indexKeyResult.getStatus()) { + log.error("The Table[" + tableName + "] index key query failed" + indexKeyResult.getMessage()); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(), " The Table[" + tableName + "] index key query failed")); + } + List<Map<String, Object>> dataList = (List<Map<String, Object>>) indexKeyResult.getData(); + List<String> dbIndexKey = StringUtil.isEmpty(dataList.get(0)) ? Lists.newArrayList() : (List<String>) dataList.get(0).get("index_key"); + List<String> schemaIndexKey = databaseService.getIndexKey(tableName); + if (schemaIndexKey.equals(dbIndexKey)) { + continue; + } + log.error("{} schema index key inconsistent with DB, schema: {}, db: {}", tableName, schemaIndexKey, dbIndexKey); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.SCHEMA_AND_TABLE_NOT_CONSISTENCY.getCode(), + String.format(CommonErrorCode.SCHEMA_AND_TABLE_NOT_CONSISTENCY.getMessage(), + String.format("%s schema index key is: %s, db index key is: %s", tableName, schemaIndexKey, dbIndexKey))); + } + } + + private BaseResult getBaseResultByEngine(SQLQueryContext queryProfile) { + return sqlSyncQueryService.executeQuery(queryProfile); + } + + /** + * 对指定数据源dbName下的schema 与数据库的字段及字段类型进行对比检查 + * + * @param + * @param tables + * @return + */ + private void checkMetadataFieldAndType(List<String> tables, Map<String, Map<String, List<String>>> resultData) { + for (String tableName : tables) { + Map schemaMap = databaseService.getSchemaInfo(FIELDS, tableName, false); + String dbName = String.valueOf(schemaMap.get(NAMESPACE)); + if (!clickHouseHttpSource.getDbName().equals(dbName) + && !druidIoHttpSource.getDbname().equals(dbName) + && !hBaseAPISource.getDbName().equals(dbName)) continue; + BaseResult baseResult = getBaseResultByEngine(SQLQueryContext.builder().option(QueryOption.REAL_TIME.getValue()).originalSQL("describe " + tableName).build()); + if (HttpStatusCodeEnum.SUCCESS.getCode() != baseResult.getStatus()) { + log.error("The Table[" + tableName + "] Structure query failed" + baseResult.getMessage()); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(), " The Table[" + tableName + "] Structure query failed")); + } + List<Map<String, Object>> dbFields = JsonPath.read(JSON.toJSONString(baseResult.getData()), "$.fields.*"); + Map<String, Object> dbFieldMap = buildDBFiedldMap(dbFields); + List<Map<String, Object>> schemaFields = (List<Map<String, Object>>) schemaMap.get(FIELDS); + Map<String, Object> schemaFieldMap = buildSchemaFiedldMap(schemaFields); + + MapDifference<String, Object> difference = Maps.difference(schemaFieldMap, dbFieldMap); + Map<String, Object> entriesOnlyOnLeft = difference.entriesOnlyOnLeft(); + if (entriesOnlyOnLeft.size() > 0) { + log.error("Schema Fields greater than Table Fields,Schema " + tableName + " exist fields :" + entriesOnlyOnLeft); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(), + "Schema Fields greater than Table Fields,Schema " + tableName + " exist fields " + entriesOnlyOnLeft)); + } + + Map<String, Object> entriesOnlyOnRight = difference.entriesOnlyOnRight(); + String fieldValidationMessage = "Schema Fields less than Table Fields:" + entriesOnlyOnRight; + if (entriesOnlyOnRight.size() > 0) { + log.warn(tableName + " " + fieldValidationMessage); + buildData(resultData, tableName, dbName, fieldValidationMessage); + } + + Map<String, MapDifference.ValueDifference<Object>> fieldTypeDiff = difference.entriesDiffering(); + if (fieldTypeDiff.size() > 0 && (clickHouseHttpSource.getDbName().equals(dbName) + || druidIoHttpSource.getDbname().equals(dbName))) { + StringBuilder sb = new StringBuilder(); + for (String field : fieldTypeDiff.keySet()) { + sb.append(field) + .append("[") + .append(fieldTypeDiff.get(field).leftValue()) + .append(" modified to ") + .append(fieldTypeDiff.get(field).rightValue()) + .append("]") + .append(", "); + } + sb.setLength(sb.length()-2); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(QGWMessageConst.SCHEMA_FIELD_TYPE_NEED_CHECK, tableName, sb)); + } + } + } + + private static Map<String, Object> buildDBFiedldMap(List<Map<String, Object>> dbFields) { + Map<String, Object> dbFieldMap = Maps.newHashMap(); + dbFields.stream().forEach(objectMap -> { + String type = StrUtil.isBlank(metaMap.get(objectMap.get("type"))) ? "string" : metaMap.get(objectMap.get("type")); + dbFieldMap.put(String.valueOf(objectMap.get("name")), type); + }); + return dbFieldMap; + } + + private static Map<String, Object> buildSchemaFiedldMap(List<Map<String, Object>> schemaFields) { + Map<String, Object> schemaFieldMap = Maps.newHashMap(); + schemaFields.stream().forEach(objectMap -> { + String type; + if (objectMap.get("type") instanceof String) { + type = String.valueOf(objectMap.get("type")); + } else { + Map map = (Map) objectMap.get("type"); + type = String.valueOf(map.get("type")).equals("array") ? String.valueOf(map.get("type")).concat("(").concat(String.valueOf(map.get("items"))).concat(")") : String.valueOf(map.get("type")); + } + schemaFieldMap.put(String.valueOf(objectMap.get("name")), type); + }); + return schemaFieldMap; + } + + private static void buildData(Map<String, Map<String, List<String>>> resultData, String tableName, String dbName, String message) { + if (!resultData.isEmpty()) { + if (resultData.containsKey(dbName)) { + if (resultData.get(dbName).containsKey(tableName)) { + Map<String, List<String>> map = resultData.get(dbName); + map.get(tableName).add(message); + } else { + Map<String, List<String>> map = resultData.get(dbName); + List<String> list = Lists.newArrayList(); + list.add(message); + map.put(tableName, list); + resultData.put(dbName, map); + } + } else { + putData(resultData, tableName, dbName, message); + } + } else { + putData(resultData, tableName, dbName, message); + } + } + + private static void putData(Map<String, Map<String, List<String>>> resultData, String tableName, String dbName, String message) { + Map<String, List<String>> map = Maps.newHashMap(); + List<String> list = Lists.newArrayList(); + list.add(message); + map.put(tableName, list); + resultData.put(dbName, map); + } + + private void checkMetadataFunctionConstraint(List<String> tables) { + Object cfg = databaseService.getCfg(PUBLIC_SCHEMA_INFO); + Map map = JSON.parseObject(String.valueOf(JSON.toJSON(cfg)), Map.class); + List<String> aggregationFunctions = JsonPath.read(map, "$.functions.aggregation[*].name"); + for (String tableName : tables) { + Map schemaMap = databaseService.getSchemaInfo(FIELDS, tableName, false); + String dbName = String.valueOf(schemaMap.get(NAMESPACE)); + if (!clickHouseHttpSource.getDbName().equals(dbName) && !druidIoHttpSource.getDbname().equals(dbName)) + continue; + List<Map<String, Object>> schemaFields = (List<Map<String, Object>>) schemaMap.get(FIELDS); + schemaFields.stream().forEach(objectMap -> { + if(JSONPath.contains(objectMap, "$.doc.constraints.aggregation_functions")){ + String str =JsonPath.read(objectMap, "$.doc.constraints.aggregation_functions"); + String funStr = str.replaceAll(" ", ""); + List<String> funList = new ArrayList<>(Arrays.asList(funStr.split(","))); + for (String fun : funList) { + if (!aggregationFunctions.contains(fun)) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), + CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(QGWMessageConst.SCHEMA_FUNCTION_CONSTRAINT_ERROR, tableName, objectMap.get("name"), fun)); + } + } + } + }); + } + } + + private Object getTableTTLInSchema(Map schemaMap) { + Object tableTTLInSchema = null; + if (StringUtil.isNotEmpty(schemaMap)) { + if (schemaMap.containsKey("doc")) { + Map<String, Object> schemaHasDoc = (Map<String, Object>) schemaMap.get("doc"); + if (schemaHasDoc.containsKey("ttl")) { + tableTTLInSchema = StringUtil.isEmpty(schemaHasDoc.get("ttl")) ? null : schemaHasDoc.get("ttl").toString(); + } + } + } + return tableTTLInSchema; + } + + private Map<String, Object> getTableDiff(String ipPort, String tableName, Object tableTTLInSchema) { + String sql = String.format(Objects.requireNonNull(env.getProperty("TABLE_TTL")), tableName, clickHouseHttpSource.getDbName()); + Map<String, String> result = executeHttpGetOfCK(ipPort, sql, clickHouseHttpSource.getSystemDBName()); + if (Integer.parseInt(result.get("status")) != HttpStatusCodeEnum.SUCCESS.getCode()) { + log.error(" query table TTL error: {}", result.toString()); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(), result.toString())); + } + Object tableTTLInDB = null; + Map<String, Object> tableDiff = Maps.newHashMap(); + Map o = (Map) JSON.parseObject(result.get("result"), Map.class); + Map<String, Object> resultMap = (Map<String, Object>) o; + for (Map<String, Object> datum : (List<Map<String, Object>>) resultMap.get("data")) { + Matcher matcher = pTTL.matcher(String.valueOf(datum.get("table_ttl"))); + tableTTLInDB = matcher.find() ? matcher.group(1) : null; + } + if (!String.valueOf(tableTTLInDB).equals(String.valueOf(tableTTLInSchema))) { + tableDiff.put("last_ttl", StringUtil.isEmpty(tableTTLInSchema) ? null : tableTTLInSchema.toString()); + tableDiff.put("used_ttl", StringUtil.isEmpty(tableTTLInDB) ? null : tableTTLInDB.toString()); + log.warn("table ttl: {} not consistency.", tableName); + } + return tableDiff; + } + + private List<Map<String, Object>> getFieldDiff(String ipPort, String + tableName, Map<String, Object> schemaMap) { + List<Map<String, Object>> fieldDiff = Lists.newArrayList(); + Map<String, String> result = executeHttpGetOfCK(ipPort, "describe " + tableName + "_local", clickHouseHttpSource.getDbName()); + if (Integer.parseInt(result.get("status")) != HttpStatusCodeEnum.SUCCESS.getCode()) { + log.error(" query table field TTL error: {}", result.toString()); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(), result.toString())); + } + List<Map<String, Object>> fields = (List<Map<String, Object>>) schemaMap.get(FIELDS); + for (Map<String, Object> field : fields) { + Object read = JsonPath.read(result, "$.result"); + List<Object> fieldList = JsonPath.read(JSONUtil.toJsonStr(JSONUtil.parseObj(read)), "$.data[?(@.name == \"" + field.get("name") + "\")].ttl_expression"); + Object fieldTTLInSchema = null; + Object fieldTTLInDB; + if (StringUtil.isNotEmpty(field.get("doc"))) { + Map<String, Object> doc = (Map<String, Object>) field.get("doc"); + fieldTTLInSchema = StringUtil.isEmpty(doc.get("ttl")) ? null : doc.get("ttl"); + } + Matcher matcher = pTTL.matcher(String.valueOf(fieldList.get(0))); + fieldTTLInDB = matcher.find() ? matcher.group(1) : null; + if (String.valueOf(fieldTTLInSchema).equals(String.valueOf(fieldTTLInDB))) { + continue; + } + log.warn("field ttl: address {} {}-{} not consistency.", ipPort, tableName, field.get("name")); + Map<String, Object> ttlDiff = Maps.newHashMap(); + ttlDiff.put("last_ttl", StringUtil.isEmpty(fieldTTLInSchema) ? null : fieldTTLInSchema.toString()); + ttlDiff.put("used_ttl", StringUtil.isEmpty(fieldTTLInDB) ? null : fieldTTLInDB.toString()); + if (fieldDiff.stream().noneMatch(o -> o.containsKey(field.get("name").toString()))) { + Map<String, Object> item = Maps.newHashMap(); + item.put(field.get("name").toString(), ttlDiff); + fieldDiff.add(item); + } + } + return fieldDiff; + } + + + private List<String> getClusterAddressOfCK() { + List<String> endpointList = new ArrayList<>(); + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL("SELECT DISTINCT concat(host_address,':','" + trafficPort + "') as endpoint FROM clusters where cluster = 'ck_cluster'").build()); + if (baseResult.isSuccess()) { + List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); + for (Map<String, Object> datum : data) { + endpointList.add(String.valueOf(datum.get("endpoint"))); + } + } + return endpointList; + } + + private Map<String, String> executeHttpGetOfCK(String ipPort, String sql, String dbName) { + String queryURL = URLUtil.normalize("http://" + ipPort + "/?"); + StringBuilder paramBuilder = new StringBuilder("user=") + .append(clickHouseHttpSource.getRealTimeAccountUserName()).append("&") + .append("password=").append(clickHouseHttpSource.getRealTimeAccountPin()).append("&") + .append("database=").append(dbName).append("&") + .append("query=").append(sql) + .append(" FORMAT JSON;"); + List<NameValuePair> values = URLEncodedUtils.parse(paramBuilder.toString(), StandardCharsets.UTF_8); + int socketTimeOut = httpConfig.getCkRealTimeAccountSocketTimeOut(); + return httpClientService.httpGet(queryURL + URLEncodedUtils.format(values, "utf-8"), socketTimeOut); + } + + private BaseResult<List<Map<String, String>>> buildResult(List<LinkedHashMap> variables,List<Map<String, String>> resultData, int totalSize) { + if (CollectionUtil.isEmpty(resultData)) { + return BaseResultGenerator.success(StrUtil.format("There are {} datasets in total, and 100% of them pass the verification, time range is: {}", totalSize, getTimeRangeByVariable(variables)), resultData); + } else { + return BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), StrUtil.format("There are {} datasets in total, and {} of them pass the verification, time range is: {}", totalSize, NumberUtil.decimalFormat("#.##%", ((totalSize - resultData.size()) * 1.00 / totalSize)), getTimeRangeByVariable(variables)), resultData); + } + } + + private Map<String, Map<String, String>> buildResult(String test, String executeEngine, List<Map> list, boolean isSaved) { + Map<String, Map<String, String>> resultMap = Maps.newLinkedHashMap(); + Map<String, String> infoMap = Maps.newLinkedHashMap(); + CSVWriter csvWriter = null; + String benchFileName = executeEngine + "_queries_" + DateUtils.getCurrentDate(DateUtils.YYYYMMDD); + if (isSaved) { + try { + String directoryPath = new File("").getCanonicalPath() + File.separator + "benchmark"; + File directory = new File(directoryPath); + if (!directory.exists()) { + directory.mkdir(); + } + String filePath = directoryPath + File.separator + benchFileName + "." + IdUtil.simpleUUID() + ".csv"; + csvWriter = new CSVWriter(new java.io.FileWriter(filePath)); + String[] headers = {"ID", "Elapsed(ms)", "Results"}; + csvWriter.writeNext(headers); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + for (Map map : list) { + BaseResult queryResult; + Stopwatch watch = Stopwatch.createStarted(); + if (String.valueOf(map.get(TYPE)).equals("sql")) { + SqlQueryRequestParam sqlQueryRequest = JSON.parseObject(String.valueOf(map.get(TEMPLATE)), SqlQueryRequestParam.class); + sqlQueryRequest.setExecutionMode(ExecutionMode.ONESHOT.getValue()); + queryResult = queryJobService.createSQLAdHocQuery(sqlQueryRequest); + } else { + DSLQueryRequestParam dslQueryRequest = JSON.parseObject(String.valueOf(map.get(TEMPLATE)), DSLQueryRequestParam.class); + dslQueryRequest.setExecutionMode(ExecutionMode.ONESHOT.getValue()); + queryResult = queryJobService.createDSLAdHocQuery(dslQueryRequest); + } + log.info("Benchmark Test, category is: {}, ID is: {}", test, map.get(IDENTIFIER_NAME)); + if (StringUtil.isEmpty(queryResult.getStatistics())) { + Map<String, Object> statistics = Maps.newLinkedHashMap(); + statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); + queryResult.setStatistics(statistics); + } + processQueryResult(benchFileName, infoMap, csvWriter, String.valueOf(map.get(IDENTIFIER_NAME)), queryResult); + } + if (csvWriter != null) { + try { + csvWriter.close(); + } catch (IOException e) { + log.error("Failed to close CSVWriter:{}", e.getMessage()); + } + } + + resultMap.put(benchFileName, buildQueryTimeMetric(infoMap)); + return resultMap; + } + + private static String getTimeRangeByVariable(List<LinkedHashMap> variables) { + String startTime = ""; + String endTime = ""; + for (LinkedHashMap variable : variables) { + if (variable.get("key").equals("start_time")) { + startTime = String.valueOf(variable.get("default")); + } + if (variable.get("key").equals("end_time")) { + endTime = String.valueOf(variable.get("default")); + } + } + return StrUtil.format("['{}'/'{}']", startTime, endTime); + } + + private Map<String, String> buildQueryTimeMetric(Map<String, String> infoMap) { + if (StringUtil.isNotEmpty(infoMap)) { + double[] values = infoMap.values().stream().mapToDouble(value -> Double.valueOf(value)).sorted().toArray(); + infoMap.put("Min", String.valueOf(CommonUtil.round(StatUtils.min(values), 2))); + infoMap.put("Mean", String.valueOf(CommonUtil.round(StatUtils.mean(values), 2))); + infoMap.put("Median", String.valueOf(CommonUtil.round(StatUtils.percentile(values, 50), 2))); + infoMap.put("P95", String.valueOf(CommonUtil.round(StatUtils.percentile(values, 95), 2))); + infoMap.put("P99", String.valueOf(CommonUtil.round(StatUtils.percentile(values, 99), 2))); + infoMap.put("MAX", String.valueOf(CommonUtil.round(StatUtils.max(values), 2))); + } + return infoMap; + } + + private void processQueryResult(String name, Map<String, String> infoMap, CSVWriter writer, + String id, BaseResult queryResult) { + int statusCode = Integer.parseInt(queryResult.getStatus().toString()); + if (statusCode == HttpStatusCodeEnum.SUCCESS.getCode()) { + if (writer != null) { + String [] data = {id, String.valueOf(queryResult.getStatistics().get("elapsed")), String.valueOf(queryResult.getData())}; + writer.writeNext(data); + } + infoMap.put(id, queryResult.getStatistics().get("elapsed") + ""); + } else if (statusCode == HttpStatusCodeEnum.GATEWAY_TIMEOUT.getCode() || statusCode == HttpStatusCodeEnum.BAD_GATEWAY.getCode()) { + infoMap.put(id, "status:" + queryResult.getStatus() + ",message:" + queryResult.getMessage()); + } else { + throw new QGWBusinessException(statusCode, CommonErrorCode.SQL_EXECUTION_EXCEPTION.getCode(), + String.format(CommonErrorCode.SQL_EXECUTION_EXCEPTION.getMessage(), name + + "," + id + "," + queryResult.getStatus() + "," + queryResult.getMessage())); + } + } + + @Override + public void setEnvironment(Environment environment) { + this.env = environment; + } +} diff --git a/src/main/java/com/mesalab/qgw/service/impl/UtilServiceImp.java b/src/main/java/com/mesalab/qgw/service/impl/UtilServiceImp.java new file mode 100644 index 00000000..b60d9784 --- /dev/null +++ b/src/main/java/com/mesalab/qgw/service/impl/UtilServiceImp.java @@ -0,0 +1,36 @@ +package com.mesalab.qgw.service.impl; + +import com.google.common.collect.Lists; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.utils.sqlparser.SQLSyntaxParserUtil; +import com.mesalab.qgw.model.job.EncryptionInfo; +import com.mesalab.qgw.service.UtilService; +import org.jasypt.util.text.BasicTextEncryptor; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service(value = "utilService") +public class UtilServiceImp implements UtilService { + + @Override + public BaseResult getCiphertext(EncryptionInfo param) { + BaseResult baseResult; + BasicTextEncryptor textEncryptor = new BasicTextEncryptor(); + textEncryptor.setPassword(param.getSalt()); + String encrypt = textEncryptor.encrypt(param.getPassword()); + Map dataMap = new HashMap(); + dataMap.put("password", param.getPassword()); + dataMap.put("encrypted_password", encrypt); + baseResult = BaseResultGenerator.success("ok", Lists.newArrayList(dataMap)); + return baseResult; + } + @Override + public BaseResult getSQLSyntaxTree(String sql) { + List list = SQLSyntaxParserUtil.syntaxParse(sql); + return BaseResultGenerator.success(list); + } +} diff --git a/src/main/java/com/mesalab/services/common/dsl/ComDSLParse.java b/src/main/java/com/mesalab/services/common/dsl/ComDSLParse.java index 14d37b53..23471706 100644 --- a/src/main/java/com/mesalab/services/common/dsl/ComDSLParse.java +++ b/src/main/java/com/mesalab/services/common/dsl/ComDSLParse.java @@ -4,26 +4,18 @@ import cn.hutool.core.date.DateException; import cn.hutool.core.date.DateUtil; import cn.hutool.core.util.NumberUtil; import cn.hutool.core.util.ObjectUtil; -import cn.hutool.crypto.digest.DigestUtil; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.mesalab.cn.entity.pojo.DSLObject; import com.mesalab.cn.entity.pojo.DSLParser; import com.mesalab.cn.enums.RangeTypeEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.common.exception.BusinessException; -import com.mesalab.knowledge.enums.MatchEnum; import com.mesalab.qgw.constant.QGWMessageConst; -import com.geedgenetworks.utils.DateUtils; import com.geedgenetworks.utils.StringUtil; -import org.joda.time.format.ISODateTimeFormat; +import com.mesalab.common.exception.CommonErrorCode; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.time.format.DateTimeFormatter; import java.util.List; -import java.util.regex.Pattern; import static java.util.stream.Collectors.joining; @@ -41,8 +33,8 @@ public class ComDSLParse extends DSLParser { } return Joiner.on(" ").join(" AND ", defField, ">=", intervals.get(0), " AND ", defField, "<", intervals.get(1)); } catch (DateException e) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.TIME_FORMAT_ERROR)); + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.TIME_FORMAT_ERROR)); } } diff --git a/src/main/java/com/mesalab/services/common/dsl/ComDSLValidate.java b/src/main/java/com/mesalab/services/common/dsl/ComDSLValidate.java index bf2f6f6e..ed0f68e2 100644 --- a/src/main/java/com/mesalab/services/common/dsl/ComDSLValidate.java +++ b/src/main/java/com/mesalab/services/common/dsl/ComDSLValidate.java @@ -2,14 +2,14 @@ package com.mesalab.services.common.dsl; import cn.hutool.core.date.DateException; import cn.hutool.core.date.DateUtil; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; import com.mesalab.knowledge.enums.MatchEnum; import com.mesalab.knowledge.enums.RangeEnum; import com.mesalab.knowledge.enums.SortEnum; import com.mesalab.qgw.constant.QGWMessageConst; import com.mesalab.qgw.exception.QGWBusinessException; import com.geedgenetworks.utils.StringUtil; +import com.mesalab.common.exception.CommonErrorCode; import org.apache.commons.lang.Validate; import org.apache.commons.lang3.EnumUtils; import org.springframework.stereotype.Component; @@ -30,46 +30,34 @@ public class ComDSLValidate { public void validation(ComDSLObject dslObject) throws QGWBusinessException { if (StringUtil.isEmpty(dslObject)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.DSL_OBJECT_IS_INVALID)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.DSL_OBJECT_IS_INVALID)); } ComDSLObject.Query query; if (StringUtil.isEmpty(query = dslObject.getQuery())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_IS_INVALID)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_IS_INVALID)); } if (StringUtil.isEmpty(query.getDataEngine())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.DATA_ENGINE_IS_INVALID)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.DATA_ENGINE_IS_INVALID)); } if (StringUtil.isEmpty(query.getDataSource())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.DATASOURCE_IS_INVALID)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.DATASOURCE_IS_INVALID)); } ComDSLObject.Query.QueryBean parameters; if (StringUtil.isEmpty(parameters = query.getParameters())) { return; } if (!isValidGranularity(parameters.getGranularity())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.GRANULARITY_VALUE_IS_INVALID)); - } - if (!isValidMatch(parameters.getMatch())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.MATCH_TYPE)); - } - if (!isValidRange(parameters.getRange())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.INTERVALS_PARAM_ERROR)); - } - if (!isValidIntervals(parameters.getIntervals())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.INTERVALS_PARAM_ERROR)); - } - if (!isValidSort(parameters.getSort())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.INTERVALS_PARAM_ERROR)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.GRANULARITY_VALUE_IS_INVALID)); } + validMatch(parameters.getMatch()); + validRange(parameters.getRange()); + validIntervals(parameters.getIntervals()); + validSort(parameters.getSort()); } private boolean isValidGranularity(String granularity) { @@ -82,84 +70,80 @@ public class ComDSLValidate { return false; } - private boolean isValidSort(List<ComDSLObject.Query.SortBean> sortList) { + private void validSort(List<ComDSLObject.Query.SortBean> sortList) { if (CollectionUtils.isEmpty(sortList)) { - return true; + return; } for (ComDSLObject.Query.SortBean sortBean : sortList) { Validate.isTrue(EnumUtils.isValidEnum(SortEnum.class, StringUtil.upperCase(sortBean.getType())), QGWMessageConst.SORT_TYPE_IS_INVALID); String fieldKey = sortBean.getFieldKey(); if (StringUtil.isBlank(fieldKey)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.SORT_FIELD_VALUES)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.SORT_FIELD_VALUES)); } } - return true; } - private boolean isValidRange(List<ComDSLObject.Query.FilterBean> rangeList) { + private void validRange(List<ComDSLObject.Query.FilterBean> rangeList) { if (CollectionUtils.isEmpty(rangeList)) { - return true; + return; } for (ComDSLObject.Query.FilterBean rangeBean : rangeList) { if (!EnumUtils.isValidEnum(RangeEnum.class, StringUtil.upperCase(rangeBean.getType()))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.RANGE_TYPE_IS_INVALID)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.RANGE_TYPE_IS_INVALID)); } String fieldKey = rangeBean.getFieldKey(); if (StringUtil.isBlank(fieldKey)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.RANGE_FIELD_VALUES)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.RANGE_FIELD_VALUES)); } } - return true; } - private boolean isValidMatch(List<ComDSLObject.Query.FilterBean> matchList) { + private void validMatch(List<ComDSLObject.Query.FilterBean> matchList) { if (CollectionUtils.isEmpty(matchList)) { - return true; + return; } for (ComDSLObject.Query.FilterBean matchBean : matchList) { if (!EnumUtils.isValidEnum(MatchEnum.class, StringUtil.upperCase(matchBean.getType()))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.MATCH_TYPE)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.MATCH_TYPE)); } if (MatchEnum.REGEX.getType().equals(matchBean.getType())) { matchBean.getFieldValues().forEach(mv -> { if (String.valueOf(mv).startsWith("$") && String.valueOf(mv).endsWith("*")) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.MATCHING_SIGN)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.MATCHING_SIGN)); } }); } } - return true; } - private boolean isValidIntervals(List<String> intervalsBean) { + private void validIntervals(List<String> intervalsBean) { try { if (CollectionUtils.isEmpty(intervalsBean)) { - return true; + return; } if (intervalsBean.size() != 1) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.INTERVALS_PARAM_ERROR)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.INTERVALS_PARAM_ERROR)); } String[] split = intervalsBean.get(0).split("/"); if (split.length != 2) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.INTERVALS_PARAM_ERROR)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.INTERVALS_PARAM_ERROR)); } for (String dateTimeStr : split) { DateUtil.parse(dateTimeStr); } - return true; } catch (DateException e) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.TIME_FORMAT_ERROR)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.TIME_FORMAT_ERROR)); } } } diff --git a/src/main/java/com/mesalab/services/common/entity/KnowledgeBaseRequest.java b/src/main/java/com/mesalab/services/common/entity/KnowledgeBaseRequest.java new file mode 100644 index 00000000..cd371620 --- /dev/null +++ b/src/main/java/com/mesalab/services/common/entity/KnowledgeBaseRequest.java @@ -0,0 +1,37 @@ +package com.mesalab.services.common.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class KnowledgeBaseRequest implements Serializable { + @JsonProperty("kb_id") + private String kbId; + @JsonProperty("name") + private String name; + @JsonProperty("format") + private String format; + @JsonProperty("category") + private String category; + @JsonProperty("is_valid") + private Integer isValid; + @JsonProperty("version") + private String version; + @JsonProperty("sha256") + private String sha256; + @JsonProperty("origin_url") + private String originUrl; + @JsonProperty("generated_time") + private String generatedTime; + @JsonProperty("last_update_time") + private String lastUpdateTime; + +} diff --git a/src/main/java/com/mesalab/services/common/enums/EntityQueryType.java b/src/main/java/com/mesalab/services/common/enums/EntityQueryType.java deleted file mode 100644 index 1788b6e3..00000000 --- a/src/main/java/com/mesalab/services/common/enums/EntityQueryType.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.mesalab.services.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @description: for datasource - * @author: tiangaochao - * @create: 2021-08-24 - */ - -@Getter -@AllArgsConstructor -public enum EntityQueryType { - ACTIVECLIENTIP("activeclientip"), - TOPSERVERIP("topserverip"), - TOPSNI("topsni"), - SUBSCRIBERIDPOOL("subscriberidpool"), - GTPC("gtpc"); - private final String type; -} diff --git a/src/main/java/com/mesalab/services/common/enums/UnstructuredDataType.java b/src/main/java/com/mesalab/services/common/enums/UnstructuredDataType.java deleted file mode 100644 index d651e8a4..00000000 --- a/src/main/java/com/mesalab/services/common/enums/UnstructuredDataType.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.mesalab.services.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -@Getter -@AllArgsConstructor -public enum UnstructuredDataType { - ALL("all"), - MAIL("mail"), - HTTP("http"), - PCAP("pcap"); - private final String type; -} diff --git a/src/main/java/com/mesalab/services/configuration/JobConfig.java b/src/main/java/com/mesalab/services/configuration/JobConfig.java index fe28d8d1..4262c92b 100644 --- a/src/main/java/com/mesalab/services/configuration/JobConfig.java +++ b/src/main/java/com/mesalab/services/configuration/JobConfig.java @@ -17,13 +17,21 @@ public class JobConfig { public static final String STATISTICS = "statistics"; public static final String JOB = "job"; + public static final String META = "meta"; + public static final String DATA = "data"; public static final String JOB_RESOURCE_PATH = "resource_path"; public static final String JOB_ID = "job_id"; public static final String COUNT = "count"; public static final String IS_DONE = "is_done"; + public static final String START_TIME = "start_time"; + public static final String END_TIME = "end_time"; public static final String DONE_PROGRESS = "done_progress"; public static final String IS_CANCELED = "is_canceled"; public static final String IS_FAILED = "is_failed"; + public static final String REASON = "reason"; + public static final String LINKS = "links"; + public static final String LINKS_STATUS = "status"; + public static final String LINKS_RESULT = "result"; public static final String DURATION_TIME = "duration_time"; public static final String LAST_QUERY_TIME = "last_query_time"; public static final String LONG_TERM_RESULT = "result"; @@ -41,15 +49,23 @@ public class JobConfig { public static final String DETAIL = "detail"; + public static final String FILTER = "filter"; public static final String QUERY_DATA_SOURCE = "query.data_source"; public static final String SAVED_QUERY = "saved_query"; - public static final String FIELD_DISCOVERY= "field_discovery"; + public static final String FIELD_DISCOVERY = "field_discovery"; + public static final String DATAPATH_PACKET_COMBINE = "datapath_telemetry_packet_combine"; + public static final String TRAFFIC_SPECTRUM_SUMMARY ="traffic-spectrum-summary"; + public static final String TRAFFIC_SPECTRUM_UNIQUE_CLIENT_AND_SERVER_IPS ="traffic-spectrum-unique-client-and-server-ips"; + public static final String TRAFFIC_SPECTRUM_APP_DISTRIBUTION ="traffic-spectrum-app-distribution"; + public static final String TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE ="traffic-spectrum-client-ip-connect-application-usage"; + public static final String TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND ="traffic-spectrum-network-throughput-trend"; public static final String LONG_TERM_COLUMN_FAMILY = "long_term"; public static final String DELIMIT = "-"; public static final String KEY_CUSTOM_FIELD_DISCOVERY_FIELDS = "custom.field_discovery.fields"; public static final String KEY_CUSTOM_FIELD_DISCOVERY_METRIC = "custom.field_discovery.metric"; + public static final String FIELD_DISCOVERY_TOPK_METRIC_PREFIX = "metric_"; public static final String KEY_CUSTOM_FIELD_DISCOVERY_METRIC_FN = "custom.field_discovery.metric.fn"; public static final String KEY_CUSTOM_FIELD_DISCOVERY_FILTER = "custom.field_discovery.filter"; public static final String KEY_CUSTOM_STATISTICS_SQL = "custom.statistics.sql"; @@ -66,16 +82,23 @@ public class JobConfig { private int responseTimeout; @NacosValue(value = "${job.execution.timeout}", autoRefreshed = true) private int executionTimeout; - @NacosValue(value = "${job.pool.corePoolSize}", autoRefreshed = true) - private int poolCorePoolSize; - @NacosValue(value = "${job.pool.maxPoolSize}", autoRefreshed = true) - private int poolMaxPoolSize; - @NacosValue(value = "${job.pool.queueCapacity}", autoRefreshed = true) - private int poolQueueCapacity; - @NacosValue(value="${job.longPolling.enabled}", autoRefreshed = true) + @NacosValue(value = "${job.longPolling.enabled}", autoRefreshed = true) private boolean longPollingEnabled; - @NacosValue(value="${job.timeSlicing.enabled}", autoRefreshed = true) + @NacosValue(value = "${job.timeSlicing.enabled}", autoRefreshed = true) private boolean timeSlicingEnabled; - @NacosValue(value="${job.timeSlicing.interval}", autoRefreshed = true) + @NacosValue(value = "${job.timeSlicing.interval}", autoRefreshed = true) private int timeSlicingInterval; + + @NacosValue(value = "${job.threadPool.lightWeight.corePoolSize}", autoRefreshed = true) + private int lightWeightCorePoolSize; + @NacosValue(value = "${job.threadPool.lightWeight.maxPoolSize}", autoRefreshed = true) + private int lightWeightMaxPoolSize; + @NacosValue(value = "${job.threadPool.lightWeight.queueCapacity}", autoRefreshed = true) + private int lightWeightQueueCapacity; + @NacosValue(value = "${job.threadPool.heavyResource.corePoolSize}", autoRefreshed = true) + private int heavyResourceCorePoolSize; + @NacosValue(value = "${job.threadPool.heavyResource.maxPoolSize}", autoRefreshed = true) + private int heavyResourceMaxPoolSize; + @NacosValue(value = "${job.threadPool.heavyResource.queueCapacity}", autoRefreshed = true) + private int heavyResourceQueueCapacity; } diff --git a/src/main/java/com/mesalab/services/configuration/JobThreadPoolCfg.java b/src/main/java/com/mesalab/services/configuration/JobThreadPoolCfg.java deleted file mode 100644 index 6080656c..00000000 --- a/src/main/java/com/mesalab/services/configuration/JobThreadPoolCfg.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.mesalab.services.configuration; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.concurrent.ThreadPoolExecutor; - -/** - * TODO - * - * @Classname ThreadPoolTaskConfig - * @Date 2021/12/1 9:20 上午 - * @Author wWei - */ -@Configuration -@EnableAsync -public class JobThreadPoolCfg { - @Autowired - JobConfig jobConfig; - private static final int keepAliveTime = 10; - private static final String threadNamePrefix = "Async-Service-"; - - @Bean("jobExecutor") - public ThreadPoolTaskExecutor getAsyncExecutor() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(jobConfig.getPoolCorePoolSize()); - executor.setMaxPoolSize(jobConfig.getPoolMaxPoolSize()); - executor.setQueueCapacity(jobConfig.getPoolQueueCapacity()); - executor.setKeepAliveSeconds(keepAliveTime); - executor.setThreadNamePrefix(threadNamePrefix); - - // 线程池对拒绝任务的处理策略 - executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); - // 初始化 - executor.initialize(); - return executor; - } -}
\ No newline at end of file diff --git a/src/main/java/com/mesalab/services/configuration/TaskConfig.java b/src/main/java/com/mesalab/services/configuration/TaskConfig.java index 4882cb4c..211caac4 100644 --- a/src/main/java/com/mesalab/services/configuration/TaskConfig.java +++ b/src/main/java/com/mesalab/services/configuration/TaskConfig.java @@ -7,8 +7,8 @@ import org.springframework.stereotype.Component; /** * TODO * - * @Classname TaskCfg - * @Date 2021/12/8 4:26 下午 + * @Classname TaskConfig + * @Date 2024/6/26 10:55 * @Author wWei */ @Data diff --git a/src/main/java/com/mesalab/services/configuration/ThreadPoolMonitor.java b/src/main/java/com/mesalab/services/configuration/ThreadPoolMonitor.java index 1a3a509d..bb0e0132 100644 --- a/src/main/java/com/mesalab/services/configuration/ThreadPoolMonitor.java +++ b/src/main/java/com/mesalab/services/configuration/ThreadPoolMonitor.java @@ -24,26 +24,32 @@ public class ThreadPoolMonitor implements InitializingBean { private static final Iterable<Tag> TAG = Collections.singletonList(Tag.of("Async", EXECUTOR_NAME)); private final ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); - @Autowired - ThreadPoolTaskExecutor jobExecutor; - @Autowired - ThreadPoolTaskExecutor taskExecutor; + ThreadPoolTaskExecutor lightWeightThreadPool; + ThreadPoolTaskExecutor heavyResourceThreadPool; + ThreadPoolTaskExecutor taskThreadPool; private final Runnable monitor = () -> { try { - Metrics.gauge("job.thread.pool.core.size", TAG, jobExecutor, ThreadPoolTaskExecutor::getCorePoolSize); - Metrics.gauge("job.thread.pool.largest.size", TAG, jobExecutor, e -> e.getThreadPoolExecutor().getLargestPoolSize()); - Metrics.gauge("job.thread.pool.max.size", TAG, jobExecutor, ThreadPoolTaskExecutor::getMaxPoolSize); - Metrics.gauge("job.thread.pool.active.size", TAG, jobExecutor, ThreadPoolTaskExecutor::getActiveCount); - Metrics.gauge("job.thread.pool.thread.count", TAG, jobExecutor, ThreadPoolTaskExecutor::getPoolSize); - Metrics.gauge("job.thread.pool.queue.size", TAG, jobExecutor, e -> e.getThreadPoolExecutor().getQueue().size()); - - Metrics.gauge("task.thread.pool.core.size", TAG, taskExecutor, ThreadPoolTaskExecutor::getCorePoolSize); - Metrics.gauge("task.thread.pool.largest.size", TAG, taskExecutor, e -> e.getThreadPoolExecutor().getLargestPoolSize()); - Metrics.gauge("task.thread.pool.max.size", TAG, taskExecutor, ThreadPoolTaskExecutor::getMaxPoolSize); - Metrics.gauge("task.thread.pool.active.size", TAG, taskExecutor, ThreadPoolTaskExecutor::getActiveCount); - Metrics.gauge("task.thread.pool.thread.count", TAG, taskExecutor, ThreadPoolTaskExecutor::getPoolSize); - Metrics.gauge("task.thread.pool.queue.size", TAG, taskExecutor, e -> e.getThreadPoolExecutor().getQueue().size()); + Metrics.gauge("light.weight.thread.pool.core.size", TAG, lightWeightThreadPool, ThreadPoolTaskExecutor::getCorePoolSize); + Metrics.gauge("light.weight.thread.pool.largest.size", TAG, lightWeightThreadPool, e -> e.getThreadPoolExecutor().getLargestPoolSize()); + Metrics.gauge("light.weight.thread.pool.max.size", TAG, lightWeightThreadPool, ThreadPoolTaskExecutor::getMaxPoolSize); + Metrics.gauge("light.weight.thread.pool.active.size", TAG, lightWeightThreadPool, ThreadPoolTaskExecutor::getActiveCount); + Metrics.gauge("light.weight.thread.pool.thread.count", TAG, lightWeightThreadPool, ThreadPoolTaskExecutor::getPoolSize); + Metrics.gauge("light.weight.thread.pool.queue.size", TAG, lightWeightThreadPool, e -> e.getThreadPoolExecutor().getQueue().size()); + + Metrics.gauge("heavy.resource.thread.pool.core.size", TAG, heavyResourceThreadPool, ThreadPoolTaskExecutor::getCorePoolSize); + Metrics.gauge("heavy.resource.thread.pool.largest.size", TAG, heavyResourceThreadPool, e -> e.getThreadPoolExecutor().getLargestPoolSize()); + Metrics.gauge("heavy.resource.thread.pool.max.size", TAG, heavyResourceThreadPool, ThreadPoolTaskExecutor::getMaxPoolSize); + Metrics.gauge("heavy.resource.thread.pool.active.size", TAG, heavyResourceThreadPool, ThreadPoolTaskExecutor::getActiveCount); + Metrics.gauge("heavy.resource.thread.pool.thread.count", TAG, heavyResourceThreadPool, ThreadPoolTaskExecutor::getPoolSize); + Metrics.gauge("heavy.resource.thread.pool.queue.size", TAG, heavyResourceThreadPool, e -> e.getThreadPoolExecutor().getQueue().size()); + + Metrics.gauge("task.thread.pool.core.size", TAG, taskThreadPool, ThreadPoolTaskExecutor::getCorePoolSize); + Metrics.gauge("task.thread.pool.largest.size", TAG, taskThreadPool, e -> e.getThreadPoolExecutor().getLargestPoolSize()); + Metrics.gauge("task.thread.pool.max.size", TAG, taskThreadPool, ThreadPoolTaskExecutor::getMaxPoolSize); + Metrics.gauge("task.thread.pool.active.size", TAG, taskThreadPool, ThreadPoolTaskExecutor::getActiveCount); + Metrics.gauge("task.thread.pool.thread.count", TAG, taskThreadPool, ThreadPoolTaskExecutor::getPoolSize); + Metrics.gauge("task.thread.pool.queue.size", TAG, taskThreadPool, e -> e.getThreadPoolExecutor().getQueue().size()); } catch (RuntimeException e) { log.error("Collect thread pool status error:", e.getMessage()); } @@ -55,7 +61,23 @@ public class ThreadPoolMonitor implements InitializingBean { } public void clearQueue() { - jobExecutor.getThreadPoolExecutor().getQueue().clear(); - taskExecutor.getThreadPoolExecutor().getQueue().clear(); + lightWeightThreadPool.getThreadPoolExecutor().getQueue().clear(); + heavyResourceThreadPool.getThreadPoolExecutor().getQueue().clear(); + taskThreadPool.getThreadPoolExecutor().getQueue().clear(); + } + + @Autowired + private void setLightWeightThreadPool(ThreadPoolTaskExecutor lightWeightThreadPool) { + this.lightWeightThreadPool = lightWeightThreadPool; + } + + @Autowired + private void setHeavyResourceThreadPool(ThreadPoolTaskExecutor heavyResourceThreadPool) { + this.heavyResourceThreadPool = heavyResourceThreadPool; + } + + @Autowired + private void setTaskThreadPool(ThreadPoolTaskExecutor taskThreadPool) { + this.taskThreadPool = taskThreadPool; } } diff --git a/src/main/java/com/mesalab/services/controller/EntityController.java b/src/main/java/com/mesalab/services/controller/EntityController.java deleted file mode 100644 index 70d9718c..00000000 --- a/src/main/java/com/mesalab/services/controller/EntityController.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.mesalab.services.controller; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.services.common.dsl.ComDSLValidate; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.common.enums.EntityQueryType; -import com.mesalab.services.service.EntityService; -import com.geedgenetworks.utils.StringUtil; -import org.apache.commons.lang3.EnumUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import javax.servlet.http.HttpServletRequest; -import java.util.Enumeration; -/** - * @author tiangaochao - * @version * - * @date 2021/8/24 15:42 下午 - */ -@RestController -@RequestMapping(value = "/entity") -public class EntityController { - private static final Log log = LogFactory.get(); - @Autowired - private EntityService entityService; - @Autowired - private ComDSLValidate comDSLValidate; - - @PostMapping(value = "/v1/", produces = "application/json") - public BaseResult EntityQuery(HttpServletRequest request, @Validated @RequestBody ComDSLObject dslObject) { - Enumeration<String> parameterNames = request.getParameterNames(); - while (parameterNames.hasMoreElements()) { - String param = parameterNames.nextElement(); - log.debug("实体推荐接口, 参数: queryString is {},params is {}", param, dslObject); - if (!EnumUtils.isValidEnum(EntityQueryType.class, StringUtil.upperCase(param)) - || StringUtil.isNotEmpty(request.getParameter(param))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"Not Support")); - } - comDSLValidate.validation(dslObject); - return entityService.getEntityInfo(param, dslObject); - } - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"Not Support")); - } -} diff --git a/src/main/java/com/mesalab/services/controller/JobController.java b/src/main/java/com/mesalab/services/controller/JobController.java deleted file mode 100644 index 89289834..00000000 --- a/src/main/java/com/mesalab/services/controller/JobController.java +++ /dev/null @@ -1,192 +0,0 @@ -package com.mesalab.services.controller; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.service.DeferredResultHolder; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.services.configuration.JobConfig; -import com.mesalab.services.service.JobService; -import com.geedgenetworks.utils.StringUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.context.request.async.DeferredResult; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.mesalab.services.configuration.JobConfig.*; - -/** - * TODO - * - * @Classname jobController - * @Date 2021/11/24 3:54 下午 - * @Author wWei - */ -@RestController -@RequestMapping(value = "/query/job") -public class JobController { - private static final Log log = LogFactory.get(); - - @Autowired - JobService jobService; - @Autowired - JobConfig jobConfig; - @Autowired - MetadataService metadataService; - - @Autowired - private DeferredResultHolder deferredResultHolder; - - @PostMapping(consumes = "application/json") - public BaseResult commit(@RequestBody HashMap<String, Object> reqBody) { - log.info("Commit Job, param is {}", reqBody); - validateJobParams(reqBody); - String type = String.valueOf(reqBody.get(JobConfig.KEY_QUERY_TYPE)); - if (JobConfig.FIELD_DISCOVERY.equals(type) || JobConfig.STATISTICS.equals(type)) { - return jobService.commitAdHocQuery(reqBody); - } else if (JobConfig.SAVED_QUERY.equals(type)) { - return jobService.commitSavedQuery(reqBody); - } - throw new BusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Support")); - } - - @GetMapping(value = "/{jobId}/field_discovery/result") - public DeferredResult<BaseResult> fieldDiscoveryResult(@PathVariable String jobId) { - log.info("Long Polling Request,Job ID is {}", jobId); - BaseResult baseResult = null; - baseResult = jobService.getResultFieldDiscovery(jobId); - DeferredResult<BaseResult> deferredResult = - deferredResultHolder.newDeferredResult(jobId, jobConfig.getResponseTimeout(), baseResult); - if (!jobConfig.isLongPollingEnabled()) { - deferredResultHolder.handleDeferredData(deferredResult, baseResult); - return deferredResult; - } - deferredResult.onTimeout(() -> { - deferredResultHolder.handleDeferredData(deferredResult, jobService.getResultFieldDiscovery(jobId)); - }); - - if (StringUtil.isNotEmpty(baseResult) - && StringUtil.isNotEmpty(baseResult.getJob()) - && Boolean.valueOf(String.valueOf(baseResult.getJob().get(JobConfig.IS_DONE)))) { - deferredResultHolder.handleDeferredData(deferredResult, baseResult); - } - return deferredResult; - } - - @GetMapping(value = {"/{jobId}/statistics/result"}) - public DeferredResult<BaseResult> statisticsResult(@PathVariable String jobId) { - log.info("Long Polling Request,Job ID is {}", jobId); - BaseResult baseResult = null; - baseResult = jobService.longTermResult(jobId); - DeferredResult<BaseResult> deferredResult = - deferredResultHolder.newDeferredResult(jobId, jobConfig.getResponseTimeout(), baseResult); - if (!jobConfig.isLongPollingEnabled()) { - deferredResultHolder.handleDeferredData(deferredResult, baseResult); - return deferredResult; - } - deferredResult.onTimeout(() -> { - deferredResultHolder.handleDeferredData(deferredResult, jobService.longTermResult(jobId)); - }); - - if (StringUtil.isNotEmpty(baseResult) - && StringUtil.isNotEmpty(baseResult.getJob()) - && Boolean.valueOf(String.valueOf(baseResult.getJob().get(JobConfig.IS_DONE)))) { - deferredResultHolder.handleDeferredData(deferredResult, baseResult); - } - return deferredResult; - } - - @GetMapping(value = "/{jobId}/saved_query/result") - public BaseResult savedQueryResult(@PathVariable String jobId) { - return jobService.getSavedQueryResult(jobId); - } - - //未实现 - @GetMapping(value = "/{jobId}/field_discovery") - public BaseResult fieldDiscoveryStatus(@PathVariable String jobId) { - throw new BusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Support")); - } - - //未实现 - @GetMapping(value = "/{jobId}/statistics") - public BaseResult statisticsStatus(@PathVariable String jobId) { - throw new BusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Support")); - } - - @GetMapping(value = "/{jobId}/saved_query") - public BaseResult savedQueryStatus(@PathVariable String jobId) { - return jobService.getSavedQueryStatus(jobId); - } - - //未实现 - @DeleteMapping(value = "/{jobId}/field_discovery") - public BaseResult cancelFieldDiscovery(@PathVariable String jobId) { - throw new BusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Support")); - } - - //未实现 - @DeleteMapping(value = "/{jobId}/statistics") - public BaseResult cancelStatistics(@PathVariable String jobId) { - throw new BusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Support")); - } - - @DeleteMapping(value = "/{jobId}/saved_query") - public BaseResult cancelSavedQuery(@PathVariable String jobId) { - log.info("Cancel Saved Query Job, ID is {}", jobId); - return jobService.cancelSavedQuery(jobId); - } - - @GetMapping(consumes = "application/json") - public BaseResult batchStatus(@RequestBody HashMap<String, Object> reqBody) { - String queryType = String.valueOf(reqBody.get(JobConfig.KEY_QUERY_TYPE)); - List<String> ids = (List<String>) reqBody.get(JobConfig.KEY_QUERY_JOB_IDS); - if (SAVED_QUERY.equals(queryType)) { - List<Map<String, Object>> result = jobService.batchSavedQueryStatus(ids); - return BaseResultGenerator.success(result); - } - throw new BusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Support")); - } - - @GetMapping(value = "/result", consumes = "application/json") - public BaseResult batchResult(@RequestBody HashMap<String, Object> reqBody) { - String queryType = String.valueOf(reqBody.get(JobConfig.KEY_QUERY_TYPE)); - List<String> ids = (List<String>) reqBody.get(JobConfig.KEY_QUERY_JOB_IDS); - if (SAVED_QUERY.equals(queryType)) { - List<Map<String, Object>> result = jobService.batchSavedQueryResult(ids); - return BaseResultGenerator.success(result); - } - throw new BusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Support")); - } - - private void validateJobParams(HashMap<String, Object> reqBody) { - if (!reqBody.containsKey(JobConfig.KEY_QUERY_TYPE)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_TYPE_NOT_NULL)); - } - if (!reqBody.containsKey(JobConfig.KEY_QUERY_DATA_SOURCE)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_DATA_SOURCE_NOT_NULL)); - } - String dataSource = String.valueOf(reqBody.get(JobConfig.KEY_QUERY_DATA_SOURCE)); - if (!metadataService.getAllTable().contains(dataSource)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_DATA_SOURCE_ERROR)); - } - } -} diff --git a/src/main/java/com/mesalab/services/controller/KBController.java b/src/main/java/com/mesalab/services/controller/KBController.java new file mode 100644 index 00000000..7fe16e4b --- /dev/null +++ b/src/main/java/com/mesalab/services/controller/KBController.java @@ -0,0 +1,157 @@ +package com.mesalab.services.controller; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.alibaba.fastjson2.JSON; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.enums.FileFormat; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.qgw.model.basic.AuditLog; +import com.mesalab.services.common.entity.KnowledgeBaseRequest; +import com.mesalab.services.service.KBService; +import com.geedgenetworks.utils.StringUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.HttpServletRequest; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; + + +@RestController +@RequestMapping(value = "/v1/knowledge_base") +public class KBController { + private static final Log log = LogFactory.get(); + @Autowired + KBService kbService; + + @PostMapping(produces = "application/json", consumes = "multipart/form-data") + @AuditLog("KBController.publishKnowledgeBase") + public BaseResult publishKnowledgeBase(@RequestParam("file") MultipartFile file, HttpServletRequest request) { + try { + KnowledgeBaseRequest KBRequest = getKnowledgeBaseRequest(request); + validateRequest(KBRequest); + log.info("Knowledge Base File publish interface:param is :{}", KBRequest.toString()); + return kbService.publishKnowledge(file, KBRequest); + } catch (Exception e) { + log.error("Knowledge Base File publish error: {}", e.getMessage()); + return BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), "Knowledge Base File publish: " + e.getMessage()); + } + } + + @PutMapping(value = {"/{kb_Id}"},consumes = "multipart/form-data") + @AuditLog("KBController.updateKnowledgeBase") + public BaseResult updateKnowledgeBase(@PathVariable(value = "kb_Id",required = false) String kbId,@RequestPart(value = "file") MultipartFile file,HttpServletRequest request) { + try { + KnowledgeBaseRequest KBRequest = getKnowledgeBaseRequest(request); + if (StringUtil.isBlank(kbId)) { + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.KB_FILE_ID_CANNOT_BE_EMPTY)); + } + if (StringUtil.isBlank(KBRequest.getVersion())) { + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.KB_FILE_VERSION_CANNOT_BE_EMPTY)); + } + KBRequest.setKbId(kbId); + log.info("Knowledge Base File update interface:param is :{}", KBRequest.toString()); + return kbService.updateKnowledge(file, KBRequest); + } catch (Exception e) { + log.error("Knowledge Base File update error: {}", e.getMessage()); + return BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), "Knowledge Base File update error: " + e.getMessage()); + } + } + + @DeleteMapping(value = "/{kb_id}",consumes = "application/x-www-form-urlencoded") + @AuditLog("KBController.deleteKnowledgeBase") + public BaseResult deleteKnowledgeBase(@PathVariable(value = "kb_id") String kbId, @RequestParam(required = false) String version) { + try { + KnowledgeBaseRequest request = new KnowledgeBaseRequest(); + if (StrUtil.isBlankIfStr(kbId)){ + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.KB_FILE_ID_CANNOT_BE_EMPTY)); + } + if (StrUtil.isNotBlank(version)){ + request.setVersion(version); + } + request.setKbId(kbId); + log.info("Knowledge Base File delete interface,param is:{}",request.toString()); + return kbService.deleteKnowledge(request); + } catch (Exception e) { + log.error("Knowledge Base File delete error: {}", e.getMessage()); + return BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), "Knowledge Base File delete error:" + e.getMessage()); + } + } + + @PutMapping(value = "/{kb_id}/status",consumes = "application/x-www-form-urlencoded") + @AuditLog("KBController.updateStatusKnowledgeBase") + public BaseResult updateStatusKnowledgeBase(@PathVariable(value = "kb_id") String kbId, HttpServletRequest request) { + try { + KnowledgeBaseRequest KBRequest = getKnowledgeBaseRequest(request); + if (StrUtil.isBlankIfStr(kbId)){ + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.KB_FILE_ID_CANNOT_BE_EMPTY)); + } + if (StrUtil.isBlank(String.valueOf(KBRequest.getIsValid()))){ + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.KB_FILE_IS_VALID_CANNOT_BE_EMPTY)); + } + KBRequest.setKbId(kbId); + log.info("Knowledge Base File update status interface,param is:{}",KBRequest.toString()); + return kbService.updateStatus(KBRequest); + } catch (Exception e) { + log.error("Knowledge Base File update status error: {}", e.getMessage()); + return BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), "Knowledge Base File update status error: " + e.getMessage()); + } + } + + @GetMapping(produces = "application/json") + @AuditLog("KBController.getList") + public BaseResult getList(@RequestParam(required = false, value = "kb_id") String kbId, @RequestParam(required = false) String category) { + try { + log.info("Knowledge Base File get list interface,param is: kb_id= {},category = {}", kbId, category); + return kbService.getList(kbId, category); + } catch (BusinessException e) { + log.error("Knowledge Base File get list error:{}", e.getMessage()); + return BaseResultGenerator.failure(HttpStatusCodeEnum.SERVER_ERROR.getCode(), "Knowledge Base File get list error: " + e.getMessage()); + } + } + + private static KnowledgeBaseRequest getKnowledgeBaseRequest(HttpServletRequest request) { + Map<String, String> paramMap = new HashMap<>(); + Enumeration<String> parameterNames = request.getParameterNames(); + while (parameterNames.hasMoreElements()) { + String paramName = parameterNames.nextElement(); + String paramValue = request.getParameter(paramName); + paramMap.put(paramName, paramValue); + } + KnowledgeBaseRequest KBRequest = JSON.parseObject(JSON.toJSONString(JSON.toJSON(paramMap)), KnowledgeBaseRequest.class); + return KBRequest; + } + private KnowledgeBaseRequest validateRequest(KnowledgeBaseRequest request){ + if (StrUtil.isBlankIfStr(request.getName())){ + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.KB_FILE_NAME_CANNOT_BE_EMPTY)); + } + if (StrUtil.isBlankIfStr(request.getFormat())){ + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.KB_FILE_FORMAT_CANNOT_BE_EMPTY)); + }else { + if (Arrays.stream(FileFormat.values()).noneMatch(o -> o.getValue().equalsIgnoreCase(request.getFormat()))){ + throw new BusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.KB_FILE_FORMAT_NOT_SUPPORT)); + } + } + if (StrUtil.isBlankIfStr(String.valueOf(request.getIsValid()))) { + request.setIsValid(1); + } + return request; + } +} diff --git a/src/main/java/com/mesalab/services/controller/KnowledgeBaseController.java b/src/main/java/com/mesalab/services/controller/KnowledgeBaseController.java deleted file mode 100644 index e5358ed2..00000000 --- a/src/main/java/com/mesalab/services/controller/KnowledgeBaseController.java +++ /dev/null @@ -1,99 +0,0 @@ -package com.mesalab.services.controller; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.services.common.entity.KnowledgeBase; -import com.mesalab.services.service.KnowledgeBaseService; -import com.geedgenetworks.utils.StringUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; - - -@RestController -@RequestMapping(value = "/knowledge_base/v1") -public class KnowledgeBaseController { - private static final Log log = LogFactory.get(); - @Autowired - KnowledgeBaseService knowledgeService; - - @GetMapping(produces = "application/json") - public BaseResult queryKnowledge() { - try { - return knowledgeService.queryKnowledge(); - } catch (BusinessException e) { - log.error("get knowledge meta error:{}", e.getMessage()); - return BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), "get knowledge meta error: " + e.getMessage()); - } - } - - - @PostMapping(produces = "application/json") - public BaseResult publishKnowledge(@RequestParam("file") MultipartFile file, KnowledgeBase knowledge) { - KnowledgeBase knowledgeBase = validateKnowledge(knowledge); - try { - return knowledgeService.publishKnowledge(file, knowledgeBase); - } catch (Exception e) { - log.error("publish knowledge error: {}", e.getMessage()); - return BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), "publish knowledge error: " + e.getMessage()); - } - } - - @PutMapping(value = {"/{id}", ""}) - public BaseResult updateKnowledge(@PathVariable(required = false) String id, @RequestPart(value = "file") MultipartFile file, KnowledgeBase knowledge) { - try { - if (StringUtil.isNotBlank(id)) { - knowledge.setId(id); - } - return knowledgeService.updateKnowledge(file, knowledge); - } catch (Exception e) { - log.error("update knowledge error: {}", e); - return BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), "update knowledge error: " + e.getMessage()); - } - } - - @DeleteMapping(value = {"/{id}", ""}) - public BaseResult deleteKnowledge(@PathVariable(required = false) String id, KnowledgeBase knowledgeBase) { - try { - return knowledgeService.deleteKnowledge(StringUtil.isNotBlank(id) ? id : knowledgeBase.getId()); - } catch (Exception e) { - log.error("delete knowledge error: {}", e.getMessage()); - return BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), "delete knowledge error:" + e.getMessage()); - } - } - - @PatchMapping(value = "/{id}") - public BaseResult toggleKnowledge(@PathVariable String id, KnowledgeBase knowledgeBase) { - try { - if (StringUtil.isNotBlank(id)) { - knowledgeBase.setId(id); - } - return knowledgeService.toggleKnowledge(knowledgeBase); - } catch (Exception e) { - log.error("toggle knowledge error: {}", e.getMessage()); - return BaseResultGenerator.failure(ResultStatusEnum.SERVER_ERROR.getCode(), "toggle knowledge error:" + e.getMessage()); - } - } - - private KnowledgeBase validateKnowledge(KnowledgeBase knowledge) { - if (StringUtil.isEmpty(knowledge.getName())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(),"Name is required"); - } - if (knowledge.getName().contains(":")) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(),"Name cannot contain ':'"); - } - if (knowledge.getName().contains(".")) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(),"Name cannot contain '.'"); - } - if (StringUtil.isEmpty(knowledge.getFormat())) { - throw new BusinessException(ResultStatusEnum.BAD_REQUEST.getCode(),ResultCodeEnum.PARAMETER_ERROR.getCode(),"Format is required"); - } - return knowledge; - } - -} diff --git a/src/main/java/com/mesalab/services/controller/RelationController.java b/src/main/java/com/mesalab/services/controller/RelationController.java deleted file mode 100644 index 6de595ac..00000000 --- a/src/main/java/com/mesalab/services/controller/RelationController.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.mesalab.services.controller; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.common.dsl.ComDSLValidate; -import com.mesalab.services.common.enums.EntityQueryType; -import com.mesalab.services.service.RelationService; -import com.geedgenetworks.utils.StringUtil; -import org.apache.commons.lang3.EnumUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import javax.servlet.http.HttpServletRequest; -import java.util.Enumeration; - -@RestController -@RequestMapping(value = "/relation") -public class RelationController { - private static final Log log = LogFactory.get(); - @Autowired - private RelationService relationService; - @Autowired - private ComDSLValidate comDSLValidate; - - @PostMapping(value = "/v1/", produces = "application/json") - public BaseResult relationQuery(HttpServletRequest request, @Validated @RequestBody ComDSLObject dslObject) { - Enumeration<String> parameterNames = request.getParameterNames(); - while (parameterNames.hasMoreElements()) { - String param = parameterNames.nextElement(); - log.debug("subscribe id 实时推荐接口, 参数: queryString is {},params is {}", param, dslObject); - if (!EnumUtils.isValidEnum(EntityQueryType.class, StringUtil.upperCase(param)) - ||StringUtil.isNotEmpty(request.getParameter(param))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"Not Support")); - } - comDSLValidate.validation(dslObject); - return relationService.getRealRelation(param, dslObject); - } - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"Not Support")); - } -} diff --git a/src/main/java/com/mesalab/services/controller/SQLDatasetController.java b/src/main/java/com/mesalab/services/controller/SQLDatasetController.java index 791d384c..a30c109d 100644 --- a/src/main/java/com/mesalab/services/controller/SQLDatasetController.java +++ b/src/main/java/com/mesalab/services/controller/SQLDatasetController.java @@ -1,22 +1,25 @@ package com.mesalab.services.controller; -import cn.hutool.core.util.StrUtil; +import cn.hutool.core.exceptions.ExceptionUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.PropertyNamingStrategy; -import com.fasterxml.jackson.databind.annotation.JsonNaming; import com.geedgenetworks.utils.StringUtil; -import com.google.common.collect.Lists; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.utils.sqlparser.AutoPeriodHelper; import com.mesalab.qgw.constant.QGWMessageConst; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.services.service.SQLDatasetService; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.AuditLog; +import com.mesalab.qgw.service.DatabaseService; +import com.mesalab.services.service.SQLDatasetService; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.parser.JJTCCJSqlParserState; +import net.sf.jsqlparser.statement.Statement; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; @@ -31,15 +34,15 @@ import java.util.*; */ @RestController @RequestMapping(value = "/v1") -public class SQLDatasetController -{ +public class SQLDatasetController { private static final Log log = LogFactory.get(); - @Autowired - SQLDatasetService sqlDatasetService; + private DatabaseService databaseService; + private SQLDatasetService sqlDatasetService; @DeleteMapping(value = "sql/query/{query_id}") + @AuditLog("DatabaseController.killQuery") public BaseResult killQuery(@PathVariable("query_id") String queryId) { - log.warn("停止查询, 参数: queryId is {}", queryId); + log.warn("skill query, param: queryId is {}", queryId); if (StringUtil.isBlank(queryId)) { return BaseResultGenerator.success4Message("ok"); } @@ -48,18 +51,22 @@ public class SQLDatasetController @RequestMapping(value = "sql/query/query_id", method = RequestMethod.POST) + @AuditLog("DatabaseController.getQueryId") public BaseResult getQueryId(@RequestParam String query, @RequestParam(value = "result_id", required = false) Integer resultId) { - log.info("获取当前SQL Query ID, 参数: query ={}，result_id ={}", query, resultId); + log.info("get current SQL Query ID, parma: query ={}，result_id ={}", query, resultId); if (StringUtil.isEmpty(query)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_CANNOT_BE_EMPTY)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_CANNOT_BE_EMPTY)); } - String queryId; - if (resultId != null) { - queryId = sqlDatasetService.getCustomQueryId(resultId.toString(), query.toString()); - } else { - queryId = sqlDatasetService.getCustomQueryId("", query.toString()); + Statement statement; + try { + statement = CCJSqlParserUtil.parse(query); + } catch (JSQLParserException e) { + log.error("Error Parsing SQL: {}", ExceptionUtil.getRootCauseMessage(e)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); } + String queryId = databaseService.getCustomQueryId(resultId == null ? null : resultId.toString(), AutoPeriodHelper.buildSqlGranularity(statement).toString()); List<Map<String, Object>> result = new ArrayList<>(); Map<String, Object> map = new HashMap<>(16); map.put("queryId", queryId); @@ -68,35 +75,21 @@ public class SQLDatasetController } @GetMapping(value = "sql/query/{query_id}/progress") + @AuditLog("DatabaseController.getProcessesByQueryId") public BaseResult getProcessesByQueryId(@PathVariable("query_id") String queryId) { - log.info("查询SQL任务执行状态, 参数: queryId is {}", queryId); + log.info("get query status, param: queryId is {}", queryId); return sqlDatasetService.getProcessesByQueryId(queryId); } - @GetMapping(value = "/dataset") - public BaseResult getDatasets(@RequestParam(value = "identifier_names",required = false) String datasetIds, @RequestParam(required = false) String category, @RequestParam(required = false) String title) - { - log.info("Get Datasets. IDs is: {}, category is: {}, title is: {}", datasetIds, category, title); - return BaseResultGenerator.success(sqlDatasetService.getDatasets(Arrays.asList(StrUtil.split(datasetIds, ",")), category, title)); - } - @GetMapping(value = "/dataset/{identifier_name}") - public BaseResult getDataset(@PathVariable("identifier_name") String identifierName) - { - log.info("Get Dataset, ID is: {}", identifierName); - return BaseResultGenerator.success(sqlDatasetService.getDataset(identifierName)); + @Autowired + public void setDatabaseService(DatabaseService databaseService) { + this.databaseService = databaseService; } - - @GetMapping(value = "/dataset/{identifier_name}/preview") - public BaseResult getPreview(@PathVariable("identifier_name") String identifierName,@RequestParam(required = false) String option) - { - return sqlDatasetService.getPreview(identifierName,option); + @Autowired + public void setSQLDatasetService(SQLDatasetService sqlDatasetService) { + this.sqlDatasetService = sqlDatasetService; } - @GetMapping(value = "/dataset/global_variable") - public BaseResult getVariable() - { - return BaseResultGenerator.success(sqlDatasetService.getVariable()); - } }
\ No newline at end of file diff --git a/src/main/java/com/mesalab/services/controller/UnstructuredController.java b/src/main/java/com/mesalab/services/controller/UnstructuredController.java deleted file mode 100644 index 6d73c285..00000000 --- a/src/main/java/com/mesalab/services/controller/UnstructuredController.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.mesalab.services.controller; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.common.dsl.ComDSLValidate; -import com.mesalab.services.common.enums.EntityQueryType; -import com.mesalab.services.common.enums.UnstructuredDataType; -import com.mesalab.services.service.UnstructuredService; -import com.geedgenetworks.utils.StringUtil; -import org.apache.commons.lang3.EnumUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import javax.servlet.http.HttpServletRequest; -import java.util.Enumeration; - -@RestController -@RequestMapping(value = "/unstructured_data") -public class UnstructuredController { - private static final Log log = LogFactory.get(); - - @Autowired - private UnstructuredService unstructuredService; - @Autowired - private ComDSLValidate comDSLValidate; - - @PostMapping(value = "/v1", produces = "application/json") - public BaseResult unstructuredDataQuery(HttpServletRequest request, @Validated @RequestBody ComDSLObject dslObject) { - Enumeration<String> parameterNames = request.getParameterNames(); - while (parameterNames.hasMoreElements()) { - String param = parameterNames.nextElement(); - log.debug("非结构化数据-文件路径列表获取接口, 参数: queryString is {},params is {}", param, dslObject); - if (!EnumUtils.isValidEnum(UnstructuredDataType.class, StringUtil.upperCase(param)) - || StringUtil.isNotEmpty(request.getParameter(param))) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Support")); - } - comDSLValidate.validation(dslObject); - return unstructuredService.getUnstructuredData(param, dslObject); - } - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),"Not Support")); - } - -} diff --git a/src/main/java/com/mesalab/services/service/EntityService.java b/src/main/java/com/mesalab/services/service/EntityService.java deleted file mode 100644 index e2257ae3..00000000 --- a/src/main/java/com/mesalab/services/service/EntityService.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.mesalab.services.service; - -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.dsl.ComDSLObject; - -import java.util.List; -import java.util.Map; - - -public interface EntityService { - - /** - * Desc: TODO - * - * @param param - * @param dslObject - * @return {@link com.mesalab.common.entity.BaseResult} - * @created by wWei - * @date 2022/6/21 4:41 下午 - */ - BaseResult getEntityInfo(String param, ComDSLObject dslObject); - - /** - * Desc: TODO - * - * @param data - * @return {@link List< Map< String, Object>>} - * @created by wWei - * @date 2022/6/21 4:41 下午 - */ - List<Map<String, Object>> mergeSNI(List<Map<String, Object>> data); - - /** - * - * @param map - * @Description 构建域名Hash表 - * @date 2022/7/8 13:56 - */ - Map<String, List<String>> buildDomainHashTable(Map<String, List<String>> map); - - /** - * - * @param domain - * @return boolean - * @Description 判断domain是否属于CDN域名列表 - * @date 2022/7/7 17:52 - */ - boolean inCDN(Map<String, List<String>> cdnMap, String domain); -} - diff --git a/src/main/java/com/mesalab/services/service/KBService.java b/src/main/java/com/mesalab/services/service/KBService.java new file mode 100644 index 00000000..556b65f9 --- /dev/null +++ b/src/main/java/com/mesalab/services/service/KBService.java @@ -0,0 +1,17 @@ +package com.mesalab.services.service; + +import com.mesalab.common.entity.BaseResult; +import com.mesalab.services.common.entity.KnowledgeBaseRequest; +import org.springframework.web.multipart.MultipartFile; + +public interface KBService { + BaseResult publishKnowledge(MultipartFile file, KnowledgeBaseRequest request); + + BaseResult updateKnowledge(MultipartFile file, KnowledgeBaseRequest request); + + BaseResult deleteKnowledge(KnowledgeBaseRequest request); + + BaseResult updateStatus(KnowledgeBaseRequest request); + + BaseResult getList(String kbId, String category); +} diff --git a/src/main/java/com/mesalab/services/service/KnowledgeBaseService.java b/src/main/java/com/mesalab/services/service/KnowledgeBaseService.java deleted file mode 100644 index 4638c4c8..00000000 --- a/src/main/java/com/mesalab/services/service/KnowledgeBaseService.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.mesalab.services.service; - -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.entity.KnowledgeBase; -import org.springframework.web.multipart.MultipartFile; - -/** - * @Author wxs - * @Date 2022/7/11 - */ -public interface KnowledgeBaseService { - BaseResult queryKnowledge(); - - BaseResult publishKnowledge(MultipartFile file, KnowledgeBase knowledge) throws Exception; - - BaseResult updateKnowledge(MultipartFile file, KnowledgeBase knowledgeBase) throws Exception; - - BaseResult deleteKnowledge(String id) throws Exception; - - BaseResult toggleKnowledge(KnowledgeBase knowledgeBase) throws Exception; -} diff --git a/src/main/java/com/mesalab/services/service/RelationService.java b/src/main/java/com/mesalab/services/service/RelationService.java deleted file mode 100644 index e5f0ce1d..00000000 --- a/src/main/java/com/mesalab/services/service/RelationService.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.mesalab.services.service; - -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.dsl.ComDSLObject; - - -public interface RelationService { - - BaseResult getRealRelation(String param, ComDSLObject dslObject); -} diff --git a/src/main/java/com/mesalab/services/service/SQLDatasetService.java b/src/main/java/com/mesalab/services/service/SQLDatasetService.java index 20b8caaf..7878c99c 100644 --- a/src/main/java/com/mesalab/services/service/SQLDatasetService.java +++ b/src/main/java/com/mesalab/services/service/SQLDatasetService.java @@ -1,12 +1,7 @@ package com.mesalab.services.service; -import com.google.common.collect.Lists; import com.mesalab.common.entity.BaseResult; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - /** * SQL数据集市管理服务 * @@ -14,8 +9,7 @@ import java.util.Map; * @Date 2023/8/10 13:54 * @Author wWei */ -public interface SQLDatasetService -{ +public interface SQLDatasetService { /** * Desc: 通过queryId查询SQL任务执行状态 @@ -29,69 +23,10 @@ public interface SQLDatasetService /** * 终止查询任务 + * * @param queryId * @return */ BaseResult deleteQueryTask(String queryId); - /** - * 自定义查询ID: 由 Catalog（数据库类型): resultID+query 组成。 - * - * @param resultId - * @param query - * @return {@link String} - * @created by wWei - * @date 2021/1/7 6:48 下午 - */ - String getCustomQueryId(String resultId, String query); - - /** - * Desc: 获取动态变量 - * - * @param - * @return {@link List<LinkedHashMap>} - * @created by wWei - * @date 2023/8/16 19:39 - */ - List<LinkedHashMap> getVariable(); - - /** - * Desc: 获取单个SQL模版 - * - * @param datasetId - * @return {@link Map<String, Object>} - * @created by wWei - * @date 2023/8/17 09:45 - */ - Map<String, Object> getDataset(String datasetId); - - /** - * Desc: 批量获取SQL模版 - * - * @param - * @return {@link Map<String, Object>} - * @created by wWei - * @date 2023/8/16 15:31 - */ - Map<String, Object> getDatasets(List<String> ids, String category, String title); - - /** - * Desc: 结果预览 - * - * @param datasetId - * @return {@link BaseResult} - * @created by wWei - * @date 2023/8/16 19:33 - */ - BaseResult getPreview(String datasetId,String option); - - /** - * Desc: 获取执行SQL - * - * @param - * @return - * @created by wanghao - * @date 2023/9/16 19:33 - */ - String buildExecSQL(List<LinkedHashMap> variables, String sql); } diff --git a/src/main/java/com/mesalab/services/service/UnstructuredService.java b/src/main/java/com/mesalab/services/service/UnstructuredService.java deleted file mode 100644 index e46f9897..00000000 --- a/src/main/java/com/mesalab/services/service/UnstructuredService.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.services.service; - -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.dsl.ComDSLObject; - -import java.util.Map; - -public interface UnstructuredService { - Map<String, Object> getUnstructuredFields(); - - BaseResult getUnstructuredData(String param, ComDSLObject dslObject); -} diff --git a/src/main/java/com/mesalab/services/service/impl/EntityServiceImp.java b/src/main/java/com/mesalab/services/service/impl/EntityServiceImp.java deleted file mode 100644 index 1ac01fd2..00000000 --- a/src/main/java/com/mesalab/services/service/impl/EntityServiceImp.java +++ /dev/null @@ -1,581 +0,0 @@ -package com.mesalab.services.service.impl; - -import cn.hutool.core.net.NetUtil; -import cn.hutool.core.util.*; -import cn.hutool.crypto.digest.DigestUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONWriter; -import com.beust.jcommander.internal.Sets; -import com.google.common.base.Splitter; -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.net.InternetDomainName; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.QueryFormatEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.model.basic.HttpConfig; -import com.mesalab.knowledge.enums.MatchEnum; -import com.mesalab.qgw.model.basic.EntityConfigSource; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.qgw.service.impl.QueryServiceImpl; -import com.mesalab.qgw.service.impl.HttpClientService; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.common.dsl.ComDSLParse; -import com.mesalab.services.common.enums.EntityQueryType; -import com.mesalab.services.common.property.SqlPropertySourceFactory; -import com.mesalab.services.service.EntityService; -import com.geedgenetworks.utils.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.EnvironmentAware; -import org.springframework.context.annotation.PropertySource; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -/** - * @author tiangaochao - * @create: 2021-08-24 - */ -@Service("entityService") -@PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) -public class EntityServiceImp implements EntityService, EnvironmentAware { - private static final Log log = LogFactory.get(); - - @Autowired - private QueryServiceImpl queryService; - @Autowired - private HttpClientService httpClientServiceTemp; - @Autowired - private HttpConfig httpConfigTemp; - @Autowired - private EntityConfigSource entityConfigSourceTemp; - @Autowired - private MetadataService metadataService; - - private static HttpClientService httpClientService; - private static HttpConfig httpConfig; - private static EntityConfigSource entityConfigSource; - - @PostConstruct - public void init() { - httpClientService = this.httpClientServiceTemp; - httpConfig = this.httpConfigTemp; - entityConfigSource = this.entityConfigSourceTemp; - } - - @Value("${server.port}") - private int serverPort; - - private Environment env; - private static String localHostAddress; - - static { - localHostAddress = NetUtil.getLocalhostStr(); - } - - @Override - public BaseResult getEntityInfo(String param, ComDSLObject comDSLObject) { - - BaseResult baseResult; - String sql; - if (param.equalsIgnoreCase(EntityQueryType.ACTIVECLIENTIP.getType())) { - setDefaultValue(comDSLObject.getQuery()); - baseResult = getActiveClientIP(comDSLObject.getQuery()); - } else if (param.equalsIgnoreCase(EntityQueryType.TOPSERVERIP.getType())) { - baseResult = getResultOfServerIP(comDSLObject.getQuery()); - } else if (param.equalsIgnoreCase(EntityQueryType.TOPSNI.getType())) { - sql = generateTopSNI(comDSLObject.getQuery(), env.getProperty("ENTITY_TOP_SNI")); - baseResult = query(sql); - - String totalSQL = generateSNITotal(comDSLObject.getQuery(), env.getProperty("ENTITY_SNI_TOTAL")); - BaseResult baseResultTotal = query(totalSQL); - int limit = StringUtil.isBlank(comDSLObject.getQuery().getLimit()) ? entityConfigSource.getTopSNIDefaultSize() : Integer.parseInt(comDSLObject.getQuery().getLimit()); - baseResult = build(limit, baseResult, baseResultTotal); - } else { - baseResult = BaseResultGenerator.failure(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), "No match queryType"); - } - if (!baseResult.isSuccess()) { - BaseResult result = JSON.parseObject(baseResult.getMessage(), BaseResult.class); - throw new QGWBusinessException(baseResult.getStatus(), baseResult.getCode(), result.getMessage()); - } - return baseResult; - } - - private BaseResult build(int limit, BaseResult baseResult, BaseResult baseResultTotal) { - if (!baseResult.isSuccess() || !baseResultTotal.isSuccess()) { - return baseResult.isSuccess() ? baseResultTotal : baseResult; - } - Stopwatch watch = Stopwatch.createStarted(); - List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); - List<Map<String, Object>> total = (List<Map<String, Object>>) baseResultTotal.getData(); - long totalBytes = StringUtil.isEmpty(total.get(0).get("bytes")) ? 0 : Long.parseLong(total.get(0).get("bytes").toString()); - long totalSessions = StringUtil.isEmpty(total.get(0).get("sessions")) ? 0 : Long.parseLong(total.get(0).get("sessions").toString()); - List<Map<String, Object>> result = mergeSNI(data); - - //sort - result.sort((o1, o2) -> { - long numThis = StringUtil.isEmpty(o2.get("sessions")) ? 0 : Long.parseLong(o2.get("sessions").toString()); - long numParam = StringUtil.isEmpty(o1.get("sessions")) ? 0 : Long.parseLong(o1.get("sessions").toString()); - if (numThis > numParam) { - return 1; - } else if (numThis < numParam) { - return -1; - } else { - return 0; - } - }); - - result = result.size() > limit ? result.subList(0, limit) : result; - - long sessionsMerge = 0; - long bytesMerge = 0; - long countCDN = 0; - - for (Map<String, Object> item : result) { - bytesMerge += StringUtil.isEmpty(item.get("bytes")) ? 0 : Long.parseLong(item.get("bytes").toString()); - sessionsMerge += StringUtil.isEmpty(item.get("sessions")) ? 0 : Long.parseLong(item.get("sessions").toString()); - if (BooleanUtil.toBoolean(String.valueOf(item.get("isCDN")))) { - countCDN++; - } - if (!StringUtil.startsWith(String.valueOf(item.get("ssl_sni")), "*") - && !StringUtil.startsWith(String.valueOf(item.get("ssl_sni")), "$")) { - item.replace("ssl_sni", "$".concat(String.valueOf(item.get("ssl_sni")))); - } - item.remove("sessions"); - item.remove("bytes"); - item.remove("isCDN"); - } - - baseResult.setMessage(StrUtil.format("Top {} SNI, {} of ssl bytes, {} of ssl sessions, {} popular CDNs", - result.size(), - totalBytes == 0 ? "-" : NumberUtil.decimalFormat("#.##%", (bytesMerge * 1.0 / totalBytes)), - totalSessions == 0 ? "-" : NumberUtil.decimalFormat("#.##%", (sessionsMerge * 1.0 / totalSessions)), - countCDN == 0 ? "-" : countCDN)); - baseResult.setData(result); - Map statistics = baseResult.getStatistics(); - statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS) + (StringUtil.isEmpty(statistics.get("elapsed")) ? 0 : Long.parseLong(statistics.get("elapsed").toString()))); - statistics.put("result_rows", result.size()); - statistics.put("result_bytes", JSON.toJSONBytes(baseResult.getData(), JSONWriter.Feature.LargeObject).length); - return baseResult; - } - - @Override - public List<Map<String, Object>> mergeSNI(List<Map<String, Object>> data) { - Map<String, List<String>> cdn = buildDomainHashTable(getCfgCDN()); - List<Map<String, Object>> dataRec = Lists.newArrayList(); - Map<String, Map<String, Object>> dataSLD = Maps.newHashMap(); - Map<String, Map<String, Object>> dataMerge = Maps.newHashMapWithExpectedSize(data.size() / 4); - Iterator<Map<String, Object>> iterator = data.iterator(); - while (iterator.hasNext()) { - Map<String, Object> datum = iterator.next(); - String value = String.valueOf(datum.get("ssl_sni")); - if (StringUtil.isBlank(value) || isTSLDomain(value)) { - continue; - } - String key = getTopPrivateDomain(value); - //1. filter exception data - if (StringUtil.isBlank(key)) { - dataRec.add(datum); - continue; - } - String domain = FormatUtils.getDomain(value); - //2. filter CDN - if (inCDN(cdn, domain)) { - datum.put("isCDN", true); - dataRec.add(datum); - continue; - } - - //3 filter TopPrivateDomain - if (isSLDDomain(domain)) { - Map<String, Object> item = dataSLD.get(key); - if (StringUtil.isNotEmpty(item)) { - long bytesBefore = StringUtil.isEmpty(item.get("bytes")) ? 0 : Long.parseLong(item.get("bytes").toString()); - long sessionsBefore = StringUtil.isEmpty(item.get("sessions")) ? 0 : Long.parseLong(item.get("sessions").toString()); - long bytesAfter = StringUtil.isEmpty(datum.get("bytes")) ? 0 : Long.parseLong(datum.get("bytes").toString()); - long sessionsAfter = StringUtil.isEmpty(datum.get("sessions")) ? 0 : Long.parseLong(datum.get("sessions").toString()); - datum.put("bytes", bytesBefore + bytesAfter); - datum.put("sessions", sessionsBefore + sessionsAfter); - } - datum.put("ssl_sni", key); - dataSLD.put(key, datum); - continue; - } - - //4. first add - if (!dataMerge.containsKey(key)) { - dataMerge.put(key, datum); - continue; - } - - //5. merge - Map<String, Object> item = dataMerge.get(key); - long bytesBefore = StringUtil.isEmpty(item.get("bytes")) ? 0 : Long.parseLong(item.get("bytes").toString()); - long sessionsBefore = StringUtil.isEmpty(item.get("sessions")) ? 0 : Long.parseLong(item.get("sessions").toString()); - long bytesAfter = StringUtil.isEmpty(datum.get("bytes")) ? 0 : Long.parseLong(datum.get("bytes").toString()); - long sessionsAfter = StringUtil.isEmpty(datum.get("sessions")) ? 0 : Long.parseLong(datum.get("sessions").toString()); - item.put("ssl_sni", "*.".concat(key)); - item.put("bytes", bytesBefore + bytesAfter); - item.put("sessions", sessionsBefore + sessionsAfter); - } - List<Map<String, Object>> result = Lists.newArrayList(); - dataMerge.keySet().forEach(o -> result.add(dataMerge.get(o))); - dataSLD.keySet().forEach(o -> result.add(dataSLD.get(o))); - result.addAll(dataRec); - return result; - } - - @Override - public Map<String, List<String>> buildDomainHashTable(Map<String, List<String>> cdn) { - Map<String, List<String>> map = Maps.newHashMap(); - for (String key : cdn.keySet()) { - List<String> list = cdn.get(key); - if (StringUtil.isEmpty(list)) { - continue; - } - list.forEach(o -> { - String topPrivateDomain = getTopPrivateDomain(o); - if (StringUtil.isEmpty(map.get(topPrivateDomain))) { - map.put(topPrivateDomain, Lists.newArrayList(o)); - } else { - map.get(topPrivateDomain).add(o); - } - }); - } - return map; - } - - @Override - public boolean inCDN(Map<String, List<String>> cdnMap, String domain) { - String topPrivateDomain = getTopPrivateDomain(domain); - List<String> list = cdnMap.get(topPrivateDomain); - if (StringUtil.isEmpty(list)) { - return false; - } - List<String> param = Splitter.on(".").omitEmptyStrings().splitToList(domain); - out: - for (String cdn : list) { - List<String> flag = Splitter.on(".").omitEmptyStrings().splitToList(cdn); - if (flag.size() > param.size()) { - continue; - } - for (int i = 1; i < flag.size() + 1; i++) { - if (!String.valueOf(param.get(param.size() - i)).equalsIgnoreCase(String.valueOf(flag.get(flag.size() - i)))) { - continue out; - } - } - return true; - } - return false; - } - - private String getTopPrivateDomain(String domain) { - String topPrivateDomain = null; - try { - topPrivateDomain = FormatUtils.getTopPrivateDomain(domain); - } catch (RuntimeException outException) { - log.warn("Parse top-level domain exceptions, exception domain names:" + domain); - } - return topPrivateDomain; - } - - public boolean isTSLDomain(String domain) { - try { - domain = FormatUtils.getDomain(domain); - if (!StringUtil.isBlank(domain) && InternetDomainName.isValid(domain)) { - InternetDomainName internetDomainName = InternetDomainName.from(domain); - return StringUtil.isEmpty(internetDomainName.publicSuffix()) ? false : internetDomainName.isPublicSuffix(); - } else { - return false; - } - } catch (RuntimeException exception) { - log.warn("valid isTSL domain exceptions, exception domain names:" + domain); - } - return false; - } - - public boolean isSLDDomain(String domain) { - try { - if (!StringUtil.isBlank(domain) && InternetDomainName.isValid(domain)) { - InternetDomainName internetDomainName = InternetDomainName.from(domain); - return internetDomainName.isTopPrivateDomain(); - } else { - return false; - } - } catch (RuntimeException exception) { - log.warn("valid isSLD domain exceptions, exception domain names:" + domain); - } - return false; - } - - - private Map<String, List<String>> getCfgCDN() { - Object codeInfo = metadataService.getCfg("public_code_info.json"); - if (StringUtil.isNotEmpty(codeInfo)) { - Map<String, Map<String, List<String>>> data = JSON.parseObject(codeInfo.toString(), Map.class); - return Maps.newHashMap(data.get("CDN")); - } - return Maps.newHashMap(); - } - - private BaseResult getActiveClientIP(ComDSLObject.Query query) { - - ComDSLObject.Query.FilterBean filterBean = query.getParameters().getMatch().get(0); - if (MatchEnum.EXACTLY.getType().equalsIgnoreCase(String.valueOf(filterBean.getType()))) { - return query(generateActiveClientIP(query, env.getProperty("ENTITY_ACTIVE_CLIENT_IP"), query.getLimit())); - } else { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.MATCH_TYPE)); - } - } - - private void setDefaultValue(ComDSLObject.Query query) { - if (StringUtil.isEmpty(query.getParameters().getIntervals())) { - Date now = new Date(); - Date someMinute = DateUtils.getSomeMinute(now, -30); - String end = DateUtils.getFormatDate(now, DateUtils.YYYY_MM_DD_HH24_MM_SS); - String start = DateUtils.getFormatDate(someMinute, DateUtils.YYYY_MM_DD_HH24_MM_SS); - List<String> list = Lists.newArrayList(start.concat("/").concat(end)); - query.getParameters().setIntervals(list); - } - query.setLimit(StringUtil.isEmpty(query.getLimit()) ? String.valueOf(entityConfigSource.getActiveClientIP()) : query.getLimit()); - } - - public boolean isValid(BaseResult baseResult, String[] intervals) { - if (StringUtil.isEmpty(baseResult.getData())) { - return false; - } - List<Map> data = (List<Map>) baseResult.getData(); - long updateTime = (long) (data.get(0).get("last_update_time")); - long paramStartTime = DateUtils.convertStringToTimestamp(intervals[0], TimeConstants.YYYY_MM_DD_HH24_MM_SS); - long paramEndTime = DateUtils.convertStringToTimestamp(intervals[1], TimeConstants.YYYY_MM_DD_HH24_MM_SS); - if (paramStartTime >= updateTime - 30 * 60 && paramEndTime <= updateTime + 30 * 60) { - return true; - } else { - return false; - } - } - - private BaseResult formatResult(BaseResult baseResult, String limit) { - if (String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(String.valueOf(baseResult.getStatus()))) { - List<Object> alterationResultList = new ArrayList<>(); - List<Map> resultList = new ObjectMapper().convertValue(baseResult.getData(), new TypeReference<List<Map>>() { - }); - resultList.forEach(r -> { - if (ObjectUtil.isNotEmpty(r)) { - List<Object> list = (List<Object>) JSON.parseObject((String)(r.get("client_ip")), Object.class); - alterationResultList.addAll(list); - } - }); - - List<Object> sortedAndDistinctList = alterationResultList.stream() - .sorted(Comparator.comparingInt(x -> Integer.parseInt(String.valueOf(((Map) x).get("sessions")))).reversed()) - .filter(distinctByKey(x -> String.valueOf(((Map) x).get("common_client_ip")))) - .collect(Collectors.toList()); - - List<Object> dataResult = new ArrayList<>(); - for (Object o : sortedAndDistinctList) { - if (dataResult.size() >= Integer.parseInt(limit)) { - break; - } - Map clientIp = new HashMap(); - clientIp.put("client_ip", ((Map) o).get("common_client_ip")); - dataResult.add(clientIp); - } - baseResult.getStatistics().put("result_rows", dataResult.size()); - baseResult.getStatistics().put("result_bytes", JSON.toJSONString(dataResult).getBytes().length); - return BaseResultGenerator.generate(baseResult.getStatus(), baseResult.getCode(), baseResult.getMessage(), dataResult, baseResult.getMeta(), baseResult.getStatistics(), QueryFormatEnum.JSON.getValue()); - } else { - baseResult = BaseResultGenerator.error(baseResult.getStatus(), baseResult.getMessage()); - } - return baseResult; - } - - /** - * @param keyExtractor - * @return java.util.function.Predicate<T> - * @Description 自定义函数，根据value去重 - * @author wanghao - * @date 2022/1/10 11:36 - */ - private static <T> Predicate<T> distinctByKey(Function<? super T, Object> keyExtractor) { - Map<Object, Boolean> seen = new ConcurrentHashMap<>(); - return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null; - } - - private BaseResult getResultOfServerIP(ComDSLObject.Query query) { - String range = ComDSLParse.parseRange(query.getParameters().getRange()); - String timeInterval = getTimeInterval(query.getParameters(), -24 * 60); - String datasource = query.getDataSource(); - String fieldWhere = timeInterval.concat(range); - - String tcpSessionSql = String.format(Objects.requireNonNull(env.getProperty("ENTITY_TCP_SESSION")), datasource, fieldWhere, - StringUtil.isBlank(query.getLimit()) ? entityConfigSource.getTopServerIPByTCPSession() : query.getLimit()); - Map<String, String> tcpSessionResult = executeQuery(tcpSessionSql); - - String tcpClientIpsSql = String.format(Objects.requireNonNull(env.getProperty("ENTITY_TCP_UNIQ_CLIENT_IPS")), datasource, fieldWhere, - StringUtil.isBlank(query.getLimit()) ? entityConfigSource.getTopServerIPByTCPUniqClientIP() : query.getLimit()); - Map<String, String> tcpClientIpResult = executeQuery(tcpClientIpsSql); - - String udpSessionSql = String.format(Objects.requireNonNull(env.getProperty("ENTITY_UDP_SESSION")), datasource, fieldWhere, - StringUtil.isBlank(query.getLimit()) ? entityConfigSource.getTopServerIPByUDPBySession() : query.getLimit()); - Map<String, String> udpSessionResult = executeQuery(udpSessionSql); - - String udpClientIpsSql = String.format(Objects.requireNonNull(env.getProperty("ENTITY_UDP_UNIQ_CLIENT_IPS")), datasource, fieldWhere, - StringUtil.isBlank(query.getLimit()) ? entityConfigSource.getTopServerIPByUDPUniqClientIP() : query.getLimit()); - Map<String, String> udpClientIpsResult = executeQuery(udpClientIpsSql); - - if (!String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(tcpSessionResult.get("status")) - || !String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(tcpClientIpResult.get("status")) - || !String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(udpSessionResult.get("status")) - || !String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(udpClientIpsResult.get("status"))) { - return BaseResultGenerator.error( - "tcp by session result:" + tcpSessionResult - + "tcp by client ips result:" + tcpClientIpResult - + "udp by session result:" + udpSessionResult - + "udp by client ips result:" + udpClientIpsResult - ); - } - - HashSet<Map<String, String>> hashSet = new HashSet<>(); - getData(udpSessionResult).forEach(hashSet::add); - getData(udpClientIpsResult).forEach(hashSet::add); - getData(tcpSessionResult).forEach(hashSet::add); - getData(tcpClientIpResult).forEach(hashSet::add); - - Set<Map<String, String>> data = Sets.newHashSet(); - if (StringUtil.isNotEmpty(query.getLimit()) && hashSet.size() > Integer.parseInt(query.getLimit())) { - for (Map<String, String> item : hashSet) { - if (data.size() >= Integer.parseInt(query.getLimit())) { - break; - } - data.add(item); - } - } else { - data = hashSet; - } - Map<String, Object> statistics = Maps.newHashMap(); - Map tcpSessionResultMap = JSON.parseObject(tcpSessionResult.get("result"), Map.class); - Map tcpSessionStatistics = (Map) tcpSessionResultMap.get("statistics"); - Map tcpClientIpResultMap = JSON.parseObject(tcpClientIpResult.get("result"), Map.class); - Map tcpClientIpStatistics = (Map) tcpClientIpResultMap.get("statistics"); - Map udpSessionResultMap = JSON.parseObject(udpSessionResult.get("result"), Map.class); - Map udpSessionStatistics = (Map) udpSessionResultMap.get("statistics"); - Map udpClientIpResultMap = (Map) JSON.parseObject(udpClientIpsResult.get("result"), Map.class); - Map udpClientIpStatistics = (Map) udpClientIpResultMap.get("statistics"); - for (Object key : tcpSessionStatistics.keySet()) { - if ("result_bytes".equals(key.toString())) { - statistics.put(key.toString(), JSON.toJSONString(data).getBytes().length); - } else if ("result_rows".equals(key.toString())) { - statistics.put(key.toString(), data.size()); - } else { - statistics.put(String.valueOf(key), Long.parseLong(tcpSessionStatistics.get(key).toString()) - + Long.parseLong(tcpClientIpStatistics.get(key).toString()) - + Long.parseLong(udpSessionStatistics.get(key).toString()) - + Long.parseLong(udpClientIpStatistics.get(key).toString())); - } - } - return BaseResultGenerator.success("ok", data, statistics); - } - - public List<Map> getData(Map<String, String> result) { - Map resultMap = JSON.parseObject(result.get("result"), Map.class); - List<Map<String, Object>> dataList = (List<Map<String, Object>>) resultMap.get("data"); - List<Map> serverIpList = Lists.newArrayList(); - for (Map item : dataList) { - Map<String, String> serverIpMap = Maps.newHashMap(); - serverIpMap.put("server_ip", String.valueOf(item.get("server_ip"))); - serverIpMap.put("vsys_id",String.valueOf(item.get("vsys_id"))); - serverIpList.add(serverIpMap); - } - return serverIpList; - } - - public String generateActiveClientIP(ComDSLObject.Query query, String fileSql, String limit) { - String range = ComDSLParse.parseRange(query.getParameters().getRange()); - String timeInterval = getTimeInterval(query.getParameters(), -30); - String matchDate = ComDSLParse.parseMath(query.getParameters().getMatch()); - return String.format(fileSql, query.getDataSource(), timeInterval, range.concat(matchDate), limit); - } - - private String getTimeInterval(ComDSLObject.Query.QueryBean parameters, int defaultInterval) { - if (StringUtil.isEmpty(parameters.getIntervals())) { - Date now = new Date(); - Date someMinute = DateUtils.getSomeMinute(now, defaultInterval); - String end = DateUtils.getFormatDate(now, DateUtils.YYYY_MM_DD_HH24_MM_SS); - String start = DateUtils.getFormatDate(someMinute, DateUtils.YYYY_MM_DD_HH24_MM_SS); - List<String> list = Lists.newArrayList(start.concat("/").concat(end)); - parameters.setIntervals(list); - } - StringBuilder timeInterval = new StringBuilder(); - String[] intervals = getIntervals(parameters.getIntervals()); - timeInterval.append("recv_time >= UNIX_TIMESTAMP('").append(intervals[0]).append("') AND recv_time< UNIX_TIMESTAMP('").append(intervals[1]).append("')"); - return timeInterval.toString(); - } - - private String[] getIntervals(List<String> intervals) { - return intervals.get(0).split("/"); - } - - public BaseResult query(String sql) { - BaseResult baseResult; - Map<String, String> result = executeQuery(sql); - if (String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(result.get("status"))) { - Map resultMap = JSON.parseObject(result.get("result"), Map.class); - List<Map> data = (List<Map>) resultMap.get("data"); - baseResult = BaseResultGenerator.success("ok", data, (Map) resultMap.get("statistics")); - } else { - baseResult = BaseResultGenerator.error(result.get("result")); - } - return baseResult; - } - - public String generateTopSNI(ComDSLObject.Query query, String fileSql) { - String range = ComDSLParse.parseRange(query.getParameters().getRange()); - String timeInterval = getTimeInterval(query.getParameters(), -24 * 60); - String datasource = query.getDataSource(); - return String.format(fileSql, datasource, timeInterval.concat(range), entityConfigSource.getTopSNIDataset()); - } - public String generateSNITotal(ComDSLObject.Query query, String fileSql) { - String range = ComDSLParse.parseRange(query.getParameters().getRange()); - String timeInterval = getTimeInterval(query.getParameters(), -24 * 60); - String datasource = query.getDataSource(); - return String.format(fileSql, datasource, timeInterval.concat(range)); - } - - private Map<String, String> executeQuery(String sql) { - try { - sql = URLEncoder.encode(sql, "utf-8").replaceAll("\\+", "%20"); - } catch (UnsupportedEncodingException e) { - log.error("sql Encode error: ", e); - } - - String queryURL = URLUtil.normalize(localHostAddress + ":" + serverPort + "/sql/?option=long_term&resultId=" + DigestUtil.md5Hex(sql) + "&query="); - int socketTimeOut = httpConfig.getCkLongTermAccountSocketTimeOut(); - return httpClientService.httpGet(queryURL + sql, socketTimeOut); - } - - @Override - public void setEnvironment(Environment environment) { - this.env = environment; - } -} diff --git a/src/main/java/com/mesalab/services/service/impl/JobExecuteService.java b/src/main/java/com/mesalab/services/service/impl/JobExecuteService.java index e3c4cb31..cbc01454 100644 --- a/src/main/java/com/mesalab/services/service/impl/JobExecuteService.java +++ b/src/main/java/com/mesalab/services/service/impl/JobExecuteService.java @@ -1,22 +1,32 @@ package com.mesalab.services.service.impl; +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.BooleanUtil; +import cn.hutool.core.util.NumberUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.jayway.jsonpath.JsonPath; import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.common.enums.QueryOptionEnum; +import com.mesalab.common.entity.DataTypeMapping; +import com.mesalab.common.enums.DBEngineType; +import com.mesalab.common.enums.QueryOption; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.utils.HazelcastInstanceMapUtil; import com.mesalab.common.utils.sqlparser.SQLFunctionUtil; import com.mesalab.common.utils.sqlparser.SQLHelper; +import com.mesalab.qgw.constant.DslIdentifierNameConst; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.service.QueryService; -import com.mesalab.qgw.service.MetadataService; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.*; +import com.mesalab.qgw.service.*; import com.mesalab.services.common.property.SqlPropertySourceFactory; import com.mesalab.services.configuration.JobConfig; import com.geedgenetworks.utils.DateUtils; @@ -36,16 +46,22 @@ import org.springframework.context.EnvironmentAware; import org.springframework.context.annotation.PropertySource; import org.springframework.core.env.Environment; import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.AsyncResult; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; +import java.io.Serializable; import java.math.RoundingMode; import java.text.DecimalFormat; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; -import static com.mesalab.services.service.impl.JobServiceImp.execute; -import static com.mesalab.services.service.impl.JobServiceImp.executeUpdate; +import static com.mesalab.services.service.impl.JobServiceImpl.execute; +import static com.mesalab.services.service.impl.JobServiceImpl.executeUpdate; /** * TODO @@ -58,43 +74,51 @@ import static com.mesalab.services.service.impl.JobServiceImp.executeUpdate; @PropertySource(value = "classpath:job-sql-template.sql", factory = SqlPropertySourceFactory.class) public class JobExecuteService implements EnvironmentAware { private static final Log log = LogFactory.get(); - @Autowired - Environment env; - @Autowired - QueryService queryService; - @Autowired - TaskExecuteService taskExecuteService; - @Autowired - MetadataService metadataService; - @Autowired - JobConfig jobCfg; + private Environment env; + private SQLSyncQueryService sqlSyncQueryService; + private TaskExecuteService taskExecuteService; + private DatabaseService databaseService; + private DSLService dslService; + private PacketCombineDslService packetCombineDslService; + private TrafficSpectrumDslService trafficSpectrumDslService; + private EngineConfigSource engineConfigSource; + private JobConfig jobCfg; + private final ThreadPoolTaskExecutor heavyResourceThreadPool; + private final ThreadPoolTaskExecutor lightWeightThreadPool; + private final ThreadPoolTaskExecutor taskThreadPool; + + public JobExecuteService(ThreadPoolTaskExecutor heavyResourceThreadPool, ThreadPoolTaskExecutor lightWeightThreadPool, ThreadPoolTaskExecutor taskThreadPool) { + this.heavyResourceThreadPool = heavyResourceThreadPool; + this.lightWeightThreadPool = lightWeightThreadPool; + this.taskThreadPool = taskThreadPool; + } - @Async("jobExecutor") - public void addExecutorFieldDiscovery(String id, HashMap<String, Object> param) { - Stopwatch watch = Stopwatch.createStarted(); + @Async("lightWeightThreadPool") + public Future<Boolean> addExecutorFieldDiscovery(DSLQueryRequestParam request) { try { - JobInfo jobInfo = new JobInfo(id); - String partitionKey = metadataService.getPartitionKey(param.get(JobConfig.QUERY_DATA_SOURCE).toString()); - Interval interval = getInterval("SELECT 1 FROM table WHERE " + param.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FILTER).toString(), partitionKey); + markJobBegin(request.getId()); + JobInfo jobInfo = new JobInfo(request.getId()); + String partitionKey = databaseService.getPartitionKey(request.getDataSource()); + Interval interval = getInterval("SELECT 1 FROM table WHERE " + request.getFilter(), partitionKey); if (jobCfg.isTimeSlicingEnabled() && interval.isValid()) { - executeSlicingFieldDiscoveryJob(jobInfo, param, interval, partitionKey); + executeSlicingFieldDiscoveryJob(request.getId(), jobInfo, request, interval, partitionKey); } else { - executeFieldDiscoveryJob(jobInfo, param, param.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FILTER).toString()); + executeFieldDiscoveryJob(request.getId(), jobInfo, request, request.getFilter()); } } catch (Exception e) { - executeUpdate(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_FIELD_VALUE")), JobConfig.DETAIL, JobConfig.IS_FAILED, id, true)); - log.error(e, "execute job error: jobId-{}", id); + markJobFailure(request.getId(), e.getMessage()); } finally { - execute(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_PROGRESS")), - id, true, 1, watch.elapsed(TimeUnit.MILLISECONDS))); + markJobCompletion(request.getId()); } + return new AsyncResult<>(true); } - @Async("jobExecutor") + @Deprecated + @Async("lightWeightThreadPool") public void addExecutorStatistics(String id, HashMap<String, Object> param, Map<String, Object> property) { JobInfo jobInfo = new JobInfo(id); try { - Interval interval = getInterval(param.get(JobConfig.KEY_CUSTOM_STATISTICS_SQL).toString(), metadataService.getPartitionKey(param.get(JobConfig.QUERY_DATA_SOURCE).toString())); + Interval interval = getInterval(param.get(JobConfig.KEY_CUSTOM_STATISTICS_SQL).toString(), databaseService.getPartitionKey(param.get(JobConfig.QUERY_DATA_SOURCE).toString())); if (jobCfg.isTimeSlicingEnabled() && interval.isValid() && StringUtil.isNotEmpty(property.get(JobConfig.JOB_PROPERTY_K_OPTION))) { @@ -118,8 +142,232 @@ public class JobExecuteService implements EnvironmentAware { } } + @Async("lightWeightThreadPool") + public void addExecutorSql(SqlQueryRequestParam request) { + try { + markJobBegin(request.getId()); + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder() + .originalSQL(request.getStatement()) + .option(request.isDryRun() ? QueryOption.SYNTAX_VALIDATION.getValue() : QueryOption.REAL_TIME.getValue()) + .format(request.getOutputMode().getValue()) + .build()); + updateJobResultOnQueryCache(request.getId(), baseResult, null); + } catch (RuntimeException e) { + markJobFailure(request.getId(), e.getMessage()); + } finally { + markJobCompletion(request.getId()); + } + } + + @Async("lightWeightThreadPool") + public void addExecutorDsl(DSLQueryRequestParam request) { + try { + markJobBegin(request.getId()); + DSLQueryContext dslProfile = BeanUtil.copyProperties(request, DSLQueryContext.class); + updateJobResultOnQueryCache(request.getId(), dslService.execDsl(dslProfile, request.isDryRun()), null); + } catch (RuntimeException e) { + markJobFailure(request.getId(), e.getMessage()); + } finally { + markJobCompletion(request.getId()); + } + } + + public BaseResult addDslExecutorPacketCombineWithoutCache(DSLQueryRequestParam request) { + try { + CompletableFuture<BaseResult> future = CompletableFuture.supplyAsync(() -> packetCombineDslService.run(request), heavyResourceThreadPool); + return future.get(); + } catch (InterruptedException | ExecutionException | RuntimeException e) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Job was Interrupted. message: " + e.getMessage())); + } + } + + public BaseResult addDslExecutorTrafficSpectrumWithoutCache(DSLQueryRequestParam request) { + try { + CompletableFuture<BaseResult> future = CompletableFuture.supplyAsync(() -> trafficSpectrumDslService.run(request), lightWeightThreadPool); + return future.get(); + } catch (InterruptedException | ExecutionException | RuntimeException e) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Job was Interrupted. message: " + e.getMessage())); + } + } + + @Async("lightWeightThreadPool") + public void addDslExecutorTrafficSpectrumWithCache(DSLQueryRequestParam request, CountDownLatch countDownLatch) { + try { + markJobBegin(request.getId()); + Interval interval = buildDSLInterval(request); + if (DslIdentifierNameConst.TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE.equalsIgnoreCase(request.getName()) + && jobCfg.isTimeSlicingEnabled() + && interval.isValid()) { + JobInfo jobInfo = new JobInfo(request.getId()); + Long end = interval.getEnd(); + Long start = interval.getStart(); + int timeSlicingInterval = jobCfg.getTimeSlicingInterval(); + long endSlicing = end; + long startSlicing = start; + while (endSlicing > start) { + if (jobInfo.isTimeout()) { + log.error("execute job timeout: job-{}", jobInfo); + markJobFailure(request.getId(), "execute job timeout"); + break; + } + QueryCache queryCache = HazelcastInstanceMapUtil.get(request.getId()); + if (StrUtil.isEmptyIfStr(queryCache)) { + log.warn("cache expiration: job-{}", jobInfo); + break; + } + Map<String, Object> job = queryCache.getBaseResult().getJob(); + if (BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_DONE))) + || BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_FAILED))) + || BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_CANCELED)))) { + log.warn("job done or interrupt: job obj-{}", job); + break; + } + startSlicing = endSlicing - timeSlicingInterval; + if (startSlicing < start) { + startSlicing = start; + } + DecimalFormat format = getDecimalFormat(); + double doneProgress = Double.parseDouble(format.format((end - startSlicing) * 1.0 / (end - start))); + DSLQueryRequestParam requestParamTemp = BeanUtil.copyProperties(request, DSLQueryRequestParam.class); + requestParamTemp.setIntervals(Lists.newArrayList( + DateUtil.format(new Date(startSlicing * 1000), DatePattern.UTC_PATTERN) + + "/" + + DateUtil.format(new Date(endSlicing * 1000), DatePattern.UTC_PATTERN))); + int sqrt = (int) Math.sqrt(engineConfigSource.getMaxCacheNum()); + requestParamTemp.setLimit(String.valueOf(Math.max(sqrt, engineConfigSource.getTrafficSpectrumClientIPAppResultNum()))); + + BaseResult baseResult = trafficSpectrumDslService.run(requestParamTemp); + if (!baseResult.isSuccess()) { + markJobFailure(request.getId(), baseResult.getMessage()); + break; + } + endSlicing = startSlicing; + BaseResult<Object> baseResultCache = queryCache.getBaseResult(); + if (baseResultCache != null && StringUtil.isNotEmpty(baseResultCache.getData())) { + if (StringUtil.isEmpty(baseResult.getData())) { + continue; + } + List<Map<String, Object>> data1 = (List<Map<String, Object>>) baseResult.getData(); + List<Map<String, Object>> data2 = (List<Map<String, Object>>) baseResultCache.getData(); + Map<String, Object> map = trafficSpectrumDslService.mergeInternalExternalBipartiteGraph(data1.get(0), data2.get(0)); + baseResultCache.setData(Lists.newArrayList(map)); + updateJobResultOnQueryCache(request.getId(), baseResultCache, doneProgress); + } else { + updateJobResultOnQueryCache(request.getId(), baseResult, doneProgress); + } + } + } else { + updateJobResultOnQueryCache(request.getId(), trafficSpectrumDslService.run(request), null); + } + } catch (RuntimeException e) { + markJobFailure(request.getId(), e.getMessage()); + } finally { + try { + markJobCompletion(request.getId()); + } finally { + if (countDownLatch != null) { + countDownLatch.countDown(); + } + } + } + } + + private Interval buildDSLInterval(DSLQueryRequestParam request) { + List<String> intervals = request.getIntervals(); + Interval interval = new Interval(); + if (intervals != null && intervals.size() == 1) { + String[] split = intervals.get(0).split("/"); + interval.setStart(DateUtil.parse(split[0]).getTime() / 1000); + interval.setEnd(DateUtil.parse(split[1]).getTime() / 1000); + } + return interval; + } + + @Async("heavyResourceThreadPool") + public void addExecutorDslPacketCombineWithCache(DSLQueryRequestParam request, CountDownLatch countDownLatch) { + try { + markJobBegin(request.getId()); + updateJobResultOnQueryCache(request.getId(), packetCombineDslService.run(request), null); + } catch (RuntimeException e) { + markJobFailure(request.getId(), e.getMessage()); + } finally { + try { + markJobCompletion(request.getId()); + } finally { + if (countDownLatch != null) { + countDownLatch.countDown(); + } + } + } + } + + private static void markJobBegin(String id) { + log.info("Async Query Job Start, id is: {}", id); + try { + HazelcastInstanceMapUtil.retrieveMap().lock(id); + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + Objects.requireNonNull(queryCache).getBaseResult().getJob().put(JobConfig.START_TIME, DateUtils.convertTimestampToString(System.currentTimeMillis() / 1000, DatePattern.UTC_PATTERN)); + HazelcastInstanceMapUtil.put(id, queryCache); + } finally { + HazelcastInstanceMapUtil.retrieveMap().unlock(id); + } + } + + public static void markJobFailure(String id, String reasonMessage) { + log.error("Async Query Job Error, message is: {}", reasonMessage); + try { + HazelcastInstanceMapUtil.retrieveMap().lock(id); + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + Map<String, Object> job = Objects.requireNonNull(queryCache).getBaseResult().getJob(); + job.put(JobConfig.IS_FAILED, true); + job.put(JobConfig.REASON, reasonMessage); + HazelcastInstanceMapUtil.put(id, queryCache); + } finally { + HazelcastInstanceMapUtil.retrieveMap().unlock(id); + } + } + + private static void markJobCompletion(String id) { + log.info("Async Query Job Done, id is: {}", id); + try { + HazelcastInstanceMapUtil.retrieveMap().lock(id); + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + Map<String, Object> job = Objects.requireNonNull(queryCache).getBaseResult().getJob(); + job.put(JobConfig.IS_DONE, true); + job.put(JobConfig.DONE_PROGRESS, 1); + job.put(JobConfig.END_TIME, DateUtils.convertTimestampToString(System.currentTimeMillis() / 1000, DatePattern.UTC_PATTERN)); + HazelcastInstanceMapUtil.put(id, queryCache); + } finally { + HazelcastInstanceMapUtil.retrieveMap().unlock(id); + } + } + + private static void updateJobResultOnQueryCache(String id, BaseResult baseResult, Double doneProgress) { + if (!baseResult.isSuccess()) { + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), baseResult.getMessage())); + } + try { + HazelcastInstanceMapUtil.retrieveMap().lock(id); + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + Map<String, Object> job = queryCache.getBaseResult().getJob(); + if (doneProgress != null) { + job.put(JobConfig.DONE_PROGRESS, doneProgress); + } + baseResult.setJob(job); + queryCache.setBaseResult(baseResult); + HazelcastInstanceMapUtil.put(id, queryCache); + } finally { + HazelcastInstanceMapUtil.retrieveMap().unlock(id); + } + } + + + @Deprecated private void executeLongTerm(JobInfo jobInfo, String cusSql, String countDataSource, String newFilter, String countFilter, boolean isSecBuild) throws JSQLParserException { - if (isCancel(jobInfo.getId())) { + if (isCancelLongTerm(jobInfo.getId())) { return; } @@ -140,42 +388,55 @@ public class JobExecuteService implements EnvironmentAware { LongValue longValue = new LongValue(Long.parseLong(limit.getRowCount().toString()) * 10); limit.setRowCount(longValue.getValue() > 10000 ? longValue : new LongValue(10000)); } - List<Future<Boolean>> list = Lists.newArrayList(taskExecuteService.executeLongTerm(jobInfo.getId(), parseSQL.toString())); - waitAsyncResultAndUpdateProcess(jobInfo, list); + List<Future<Boolean>> list = Lists.newArrayList(CompletableFuture.supplyAsync(() -> taskExecuteService.executeLongTerm(jobInfo.getId(), parseSQL.toString()))); + waitAsyncResultAndUpdateProcessLongTerm(jobInfo, list); } - private void executeSlicingFieldDiscoveryJob(JobInfo jobInfo, Map<String, Object> param, Interval interval, String partitionKey) throws JSQLParserException { + private void executeSlicingFieldDiscoveryJob(String id, JobInfo jobInfo, DSLQueryRequestParam request, Interval interval, String partitionKey) throws ExecutionException, InterruptedException { jobInfo.setTotalTimes((int) ((interval.getEnd() - interval.getStart()) / jobCfg.getTimeSlicingInterval() + 1)); while (true) { jobInfo.setCurrentTimes(jobInfo.getCurrentTimes() + 1); if (jobInfo.isTimeout()) { log.error("execute job timeout: job-{}", jobInfo); - executeUpdate(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_FIELD_VALUE")), JobConfig.DETAIL, JobConfig.IS_FAILED, jobInfo.getId(), true)); + markJobFailure(id, "execute job timeout"); + break; + } + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + if (StrUtil.isEmptyIfStr(queryCache)) { + log.warn("cache expiration: job-{}", jobInfo); + break; + } + Map<String, Object> job = queryCache.getBaseResult().getJob(); + if (BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_DONE))) + || BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_FAILED))) + || BooleanUtil.toBoolean(String.valueOf(job.get(JobConfig.IS_CANCELED)))) { + log.warn("job done or interrupt: job obj-{}", job); break; } long start = interval.getEnd() - jobCfg.getTimeSlicingInterval(); if (start > interval.getStart()) { interval.setEnd(start); String newFilter = String.format("(%s) AND ( %s > %s AND %s <= %s)", - param.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FILTER), + request.getFilter(), partitionKey, interval.getEnd(), partitionKey, interval.getEnd() + jobCfg.getTimeSlicingInterval()); - executeFieldDiscoveryJob(jobInfo, param, newFilter); + executeFieldDiscoveryJob(id, jobInfo, request, newFilter); } else { String where = String.format("(%s) AND (%s >= %s AND %s <= %s)", - param.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FILTER), + request.getFilter(), partitionKey, interval.getStart(), partitionKey, interval.getEnd()); - executeFieldDiscoveryJob(jobInfo, param, where); + executeFieldDiscoveryJob(id, jobInfo, request, where); break; } } } + @Deprecated private void executeSlicingStatisticsJob(JobInfo jobInfo, Map<String, Object> param, Interval interval) throws JSQLParserException { - String partitionKey = metadataService.getPartitionKey(param.get(JobConfig.QUERY_DATA_SOURCE).toString()); + String partitionKey = databaseService.getPartitionKey(param.get(JobConfig.QUERY_DATA_SOURCE).toString()); List<Expression> timeFloorWithFill = SQLHelper.getFunctionParams(param.get(JobConfig.KEY_CUSTOM_STATISTICS_SQL).toString(), SQLFunctionUtil.TIME_FLOOR_WITH_FILL); String per = StringUtil.isEmpty(timeFloorWithFill) ? "PT5M" : timeFloorWithFill.get(1).toString().substring(1, timeFloorWithFill.get(1).toString().length() - 1); - String function = SQLFunctionUtil.translateCommonFun(SQLFunctionUtil.TIME_FLOOR_WITH_FILL + "(" + interval.getEnd() + ", '" + per + "', 'zero')", DBTypeEnum.CLICKHOUSE.getValue()); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(StrUtil.format("SELECT {} AS stat_time FROM {} LIMIT 1", function, param.get(JobConfig.QUERY_DATA_SOURCE))).build()); + String function = SQLFunctionUtil.translateCommonFun(SQLFunctionUtil.TIME_FLOOR_WITH_FILL + "(" + interval.getEnd() + ", '" + per + "', 'zero')", DBEngineType.CLICKHOUSE.getValue()); + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(StrUtil.format("SELECT {} AS stat_time FROM {} LIMIT 1", function, param.get(JobConfig.QUERY_DATA_SOURCE))).build()); List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); long statTime = Long.parseLong(data.get(0).get("stat_time").toString()) * 1000; @@ -185,7 +446,7 @@ public class JobExecuteService implements EnvironmentAware { //分片粒度 Max(per, TimeSlicingInterval) long perSeconds = (dateList.get(0).getTime() - dateList.get(1).getTime()) / 1000; long step = jobCfg.getTimeSlicingInterval() / perSeconds == 0 ? 1 : jobCfg.getTimeSlicingInterval() / perSeconds; - int totalTimes = new Double(Math.ceil((dateList.size() - 1) * 1.0 / step)).intValue(); + int totalTimes = Double.valueOf(Math.ceil((dateList.size() - 1) * 1.0 / step)).intValue(); jobInfo.setTotalTimes(totalTimes); while (true) { if (jobInfo.isTimeout()) { @@ -208,68 +469,260 @@ public class JobExecuteService implements EnvironmentAware { } } - private void executeFieldDiscoveryJob(JobInfo jobInfo, Map<String, Object> param, String newFilter) throws JSQLParserException { + private void executeFieldDiscoveryJob(String id, JobInfo jobInfo, DSLQueryRequestParam request, String segmentFilter) throws ExecutionException, InterruptedException { if (isCancel(jobInfo.getId())) { return; } - String dataSource = param.get(JobConfig.QUERY_DATA_SOURCE).toString(); + String dataSource = request.getDataSource(); String totalMetric = "count(*)"; - Object metric = param.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC); + Object metric = request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC); if (StringUtil.isNotEmpty(metric)) { - Object fn = param.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC_FN); - String measurements = metadataService.getValueByKeyInSchemaDoc(dataSource, "measurements"); - List<Map<String, String>> read = JsonPath.read(measurements, "$.aggregates['" + metric + "'][?(@.fn == '" + fn + "')]"); + Object fn = request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC_FN); + String measurements = databaseService.getValueByKeyInSchemaDoc(dataSource, "measurements"); + List<Map<String, String>> read = JsonPath.read(measurements, "$.field_discovery_metric['" + metric + "'][?(@.fn == '" + fn + "')]"); if (!read.isEmpty()) { String column = read.get(0).get("column"); totalMetric = read.get(0).get("fn") + "(" + column + ")"; } } - Map<String, Long> map = getCount(dataSource, newFilter, totalMetric); + Map<String, Long> map = getCount(dataSource, segmentFilter, totalMetric); long currentCount = map.get("logCount"); if (currentCount == 0) { return; } - long count = jobInfo.getCount() + currentCount; - executeUpdate(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_FIELD_VALUE")), JobConfig.DETAIL, JobConfig.COUNT, jobInfo.getId(), count)); - jobInfo.setCount(count); - List<Future<Boolean>> list = Lists.newArrayList(); - List<String> fields = (List<String>) param.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS); + Map<String, Future<List<Map<String, Object>>>> taskCallbackList = Maps.newHashMap(); + List<String> fields = (List<String>) request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS); + for (String field : fields) { - list.add(taskExecuteService.executeFieldDiscovery(jobInfo.getId(), param.get(JobConfig.QUERY_DATA_SOURCE).toString(), newFilter, field, - param.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC), param.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC_FN), map.get("totalMetric"))); + taskCallbackList.put(field, CompletableFuture.supplyAsync(() -> taskExecuteService.executeFieldDiscovery(id, request.getDataSource(), field + , request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC) + , request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC_FN) + , segmentFilter), taskThreadPool)); } - waitAsyncResultAndUpdateProcess(jobInfo, list); + waitAsyncResultAndUpdateProcess(id, jobInfo, taskCallbackList, request, map); } - private void waitAsyncResultAndUpdateProcess(JobInfo jobInfo, List<Future<Boolean>> taskList) { - double taskCompletedProcess = 0; + private void waitAsyncResultAndUpdateProcess(String id, JobInfo jobInfo, Map<String, Future<List<Map<String, Object>>>> taskCallbackList, DSLQueryRequestParam request, Map<String, Long> mapCurrent) throws ExecutionException, InterruptedException { + List<String> updatedResult = Lists.newArrayList(); while (true) { if (jobInfo.isTimeout()) { - log.error("execute job timeout: job-{}", jobInfo); - executeUpdate(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_FIELD_VALUE")), JobConfig.DETAIL, JobConfig.IS_FAILED, jobInfo.getId(), true)); - taskList.forEach(o -> o.cancel(true)); + log.error("execute job timeout, job id is: {}", id); + markJobFailure(id, "execute job timeout"); + taskCallbackList.keySet().forEach(o -> taskCallbackList.get(o).cancel(true)); break; } - long completedTaskCount = taskList.stream().filter(Future::isDone).count(); + if (updatedResult.size() >= taskCallbackList.size()) { + break; + } + long completedTaskCount = taskCallbackList.keySet().stream().filter(o -> taskCallbackList.get(o).isDone()).count(); + if ((completedTaskCount - updatedResult.size()) * 1.0 / taskCallbackList.size() < 0.01 && completedTaskCount < taskCallbackList.size()) { + continue; + } + log.info("id :{}, done size: {}, task size: {}", id, completedTaskCount, taskCallbackList.size()); DecimalFormat format = getDecimalFormat(); + double jobCompletedProcess = Double.parseDouble(format.format(jobInfo.getCurrentTimes() * 1.0 / jobInfo.getTotalTimes() - ((1 - completedTaskCount * 1.0 / taskCallbackList.size()) / jobInfo.getTotalTimes()))); + List<String> queryDoneField = taskCallbackList.keySet().stream().filter(o -> taskCallbackList.get(o).isDone()).collect(Collectors.toList()); + for (String field : queryDoneField) { + if (updatedResult.contains(field)) { + continue; + } + updatedResult.add(field); + Long currentTotalMetric = mapCurrent.get("totalMetric"); + String fnDefault = "count"; + String value = "count"; + Object metric = request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC); + Object fn = request.getCustomRequestParam().get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC_FN); + if (StringUtil.isNotEmpty(metric)) { + String measurements = databaseService.getValueByKeyInSchemaDoc(request.getDataSource(), "measurements"); + List<Map<String, String>> read = JsonPath.read(measurements, "$.field_discovery_metric['" + metric + "'][?(@.fn == '" + fn + "')]"); + if (!read.isEmpty()) { + fnDefault = fn.toString(); + value = read.get(0).get("value"); + } + } + Future<List<Map<String, Object>>> listFuture = taskCallbackList.get(field); + List<Map<String, Object>> data = listFuture.get(); + data.forEach(x -> { + for (String k : x.keySet()) { + if (k.startsWith(JobConfig.FIELD_DISCOVERY_TOPK_METRIC_PREFIX)) { + Object v = x.get(k); + x.put(k.replace(JobConfig.FIELD_DISCOVERY_TOPK_METRIC_PREFIX, ""), v); + x.remove(k); + } + } + }); + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + if (StrUtil.isEmptyIfStr(queryCache)) { + markJobFailure(id, "Cache expiration."); + log.error("field_discovery task query lastTopK Error: Cache expiration."); + return; + } + Object lastTotalMetric = 0; + if (StringUtil.isNotEmpty(queryCache.getBaseResult().getData())) { + List<Map> map = (List<Map>) queryCache.getBaseResult().getData(); + for (Map map1 : map) { + if (CollectionUtil.isNotEmpty(map1) && map1.containsKey(field) && !StrUtil.isEmptyIfStr(map1.get(field))) { + Map item = (Map) map1.get(field); + List<Map<String, Object>> topk = (List<Map<String, Object>>) item.get("topk"); + lastTotalMetric = item.get(value); + Map<String, String> metrics = Maps.newHashMap(); + metrics.put(value, "count".equalsIgnoreCase(fnDefault) ? "sum" : fnDefault); + data = mergeData(data, topk, Lists.newArrayList("value"), metrics); + } + + } + } + data = sortDataAndSetMaxSize(data, value, DataTypeMapping.LONG, false); + updateResult(id, field, currentTotalMetric, lastTotalMetric, value, data, jobCompletedProcess); - double jobCompletedProcess = Double.parseDouble(format.format(jobInfo.getCurrentTimes() * 1.0 / jobInfo.getTotalTimes() - ((1 - completedTaskCount * 1.0 / taskList.size()) / jobInfo.getTotalTimes()))); - if (jobCompletedProcess - taskCompletedProcess >= 0.1 && jobCompletedProcess < 1) { - taskCompletedProcess = jobCompletedProcess; - execute(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_PROGRESS")), - jobInfo.getId(), false, jobCompletedProcess, System.currentTimeMillis() - jobInfo.startTime)); } - if (completedTaskCount >= taskList.size()) { + if (completedTaskCount >= taskCallbackList.size()) { break; } } } + private void updateResult(String id, String field, long currentTotalMetric, Object lastTotalMetric, String value, List<Map<String, Object>> topList, double jobCompletedProcess) { + Map<String, Object> map = Maps.newHashMap(); + map.put("topk", topList); + map.put("distinct_count", Math.min(topList.size(), 100)); + map.put(value, currentTotalMetric + Long.parseLong(lastTotalMetric.toString())); + try { + HazelcastInstanceMapUtil.retrieveMap().lock(id); + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + BaseResult<Object> baseResult = queryCache.getBaseResult(); + if (jobCompletedProcess < 1) { + Map<String, Object> job = queryCache.getBaseResult().getJob(); + job.put(JobConfig.DONE_PROGRESS, jobCompletedProcess); + } + + Object data1 = baseResult.getData(); + if (StrUtil.isEmptyIfStr(data1)) { + List<Object> list = Lists.newArrayList(); + Map<Object, Object> item = Maps.newLinkedHashMap(); + item.put(field, map); + list.add(item); + baseResult.setData(list); + HazelcastInstanceMapUtil.put(id, queryCache); + return; + } + List<Map<String, Object>> list = (List<Map<String, Object>>) data1; + for (int i = 0; i < list.size(); i++) { + Map<String, Object> map1 = list.get(i); + if (map1.containsKey(field)) { + map1.put(field, map); + break; + } + } + if (list.stream().noneMatch(o -> o.containsKey(field))) { + Map<String, Object> objectObjectHashMap = Maps.newHashMap(); + objectObjectHashMap.put(field, map); + list.add(objectObjectHashMap); + } + HazelcastInstanceMapUtil.put(id, queryCache); + } finally { + HazelcastInstanceMapUtil.retrieveMap().unlock(id); + } + } + + private List<Map<String, Object>> sortDataAndSetMaxSize(List<Map<String, Object>> data, String sortElement, String sortType, boolean isAsc) { + if (DataTypeMapping.INT.equalsIgnoreCase(sortType) || DataTypeMapping.LONG.equalsIgnoreCase(sortType)) { + data.sort(Comparator.comparing(o -> Long.valueOf(StringUtil.isEmpty(o.get(sortElement)) ? Long.MIN_VALUE + "" : o.get(sortElement).toString()))); + } else if (DataTypeMapping.FLOAT.equalsIgnoreCase(sortType) || DataTypeMapping.DOUBLE.equalsIgnoreCase(sortType)) { + data.sort(Comparator.comparing(o -> Double.valueOf(StringUtil.isEmpty(o.get(sortElement)) ? Long.MIN_VALUE + "" : o.get(sortElement).toString()))); + } else { + data.sort(Comparator.comparing(o -> String.valueOf(o.get(sortElement)))); + } + if (!isAsc) { + Collections.reverse(data); + } + return data.size() > engineConfigSource.getMaxCacheNum() ? data.subList(0, engineConfigSource.getMaxCacheNum()) : data; + } + + private List<Map<String, Object>> mergeData(List<Map<String, Object>> data1, List<Map<String, Object>> date2, List<String> dimensions, Map<String, String> metrics) { + Collection<Map<String, Object>> data = CollectionUtil.addAll(data1, date2); + Map<String, Map<String, Object>> result = Maps.newHashMap(); + for (Map<String, Object> datum : data) { + StringBuilder key = new StringBuilder(); + dimensions.forEach(k -> key.append(datum.get(k))); + if (!result.containsKey(key.toString())) { + result.put(key.toString(), datum); + continue; + } + Map<String, Object> item = result.get(key.toString()); + for (String k : metrics.keySet()) { + String action = metrics.get(k); + Object o1 = item.get(k); + Object o2 = datum.get(k); + if (StringUtil.isEmpty(o1) || StringUtil.isEmpty(o2)) { + item.put(k, (StringUtil.isEmpty(o1) && StringUtil.isEmpty(o2)) ? null : StringUtil.isEmpty(o1) ? o2 : o1); + continue; + } + Number number1 = NumberUtil.parseNumber(o1.toString()); + Number number2 = NumberUtil.parseNumber(o2.toString()); + if (action.equals("sum")) { + item.put(k, NumberUtil.add(number1, number2)); + continue; + } + if (NumberUtil.isDouble(o1.toString()) || NumberUtil.isDouble(o2.toString())) { + double v1 = Double.parseDouble(number1.toString()); + double v2 = Double.parseDouble(number2.toString()); + switch (action) { + case "max": + item.put(k, NumberUtil.max(v1, v2)); + break; + case "min": + item.put(k, NumberUtil.min(v1, v2)); + break; + case "avg": + item.put(k, NumberUtil.div(NumberUtil.add(v1, v2), 2)); + break; + } + } else if (NumberUtil.isLong(o1.toString()) || NumberUtil.isLong(o2.toString())) { + long v1 = Long.parseLong(number1.toString()); + long v2 = Long.parseLong(number2.toString()); + switch (action) { + case "max": + item.put(k, NumberUtil.max(v1, v2)); + break; + case "min": + item.put(k, NumberUtil.min(v1, v2)); + break; + case "avg": + item.put(k, NumberUtil.div(NumberUtil.add(v1, v2), 2)); + break; + } + } else { + log.error("task merge data error, data is: {}, {}", data1, date2); + throw new BusinessException("task merge data error."); + } + } + } + return new ArrayList<>(result.values()); + } + private boolean isCancel(String id) { - QueryProfile build = QueryProfile.builder().query(String.format(Objects.requireNonNull(env.getProperty("FIELD_RESULT")), JobConfig.DETAIL, JobConfig.LAST_QUERY_TIME, id)). - option(QueryOptionEnum.REAL_TIME.getValue()).build(); - BaseResult baseResult = queryService.executeQuery(build); + QueryCache queryCache = HazelcastInstanceMapUtil.get(id); + if (queryCache == null) { + log.info("Can't find the cache, job id is: {}", id); + return true; + } + long currentTime = System.currentTimeMillis(); + if (currentTime - queryCache.getLatestQueryTimeMs() > jobCfg.getInteractiveTimeout()) { + queryCache.getBaseResult().getJob().put(JobConfig.IS_CANCELED, true); + HazelcastInstanceMapUtil.put(id, queryCache); + log.info("Interactive timeout, job id is: {}", id); + return true; + } + return false; + } + + @Deprecated + private boolean isCancelLongTerm(String id) { + SQLQueryContext build = SQLQueryContext.builder().originalSQL(String.format(Objects.requireNonNull(env.getProperty("FIELD_RESULT")), JobConfig.DETAIL, JobConfig.LAST_QUERY_TIME, id)). + option(QueryOption.REAL_TIME.getValue()).build(); + BaseResult baseResult = sqlSyncQueryService.executeQuery(build); if (baseResult.isSuccess() && (StringUtil.isNotEmpty(baseResult.getData()))) { List<Map<String, Object>> list = (List<Map<String, Object>>) baseResult.getData(); long statTime = Long.parseLong(list.get(0).get("value").toString()); @@ -281,10 +734,36 @@ public class JobExecuteService implements EnvironmentAware { return false; } + @Deprecated + private void waitAsyncResultAndUpdateProcessLongTerm(JobInfo jobInfo, List<Future<Boolean>> taskList) { + double taskCompletedProcess = 0; + while (true) { + if (jobInfo.isTimeout()) { + log.error("execute job timeout: job-{}", jobInfo); + executeUpdate(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_FIELD_VALUE")), JobConfig.DETAIL, JobConfig.IS_FAILED, jobInfo.getId(), true)); + taskList.forEach(o -> o.cancel(true)); + break; + } + long completedTaskCount = taskList.stream().filter(Future::isDone).count(); + DecimalFormat format = getDecimalFormat(); + + double jobCompletedProcess = Double.parseDouble(format.format(jobInfo.getCurrentTimes() * 1.0 / jobInfo.getTotalTimes() - ((1 - completedTaskCount * 1.0 / taskList.size()) / jobInfo.getTotalTimes()))); + if (jobCompletedProcess - taskCompletedProcess >= 0.1 && jobCompletedProcess < 1) { + taskCompletedProcess = jobCompletedProcess; + execute(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_PROGRESS")), + jobInfo.getId(), false, jobCompletedProcess, System.currentTimeMillis() - jobInfo.startTime)); + } + if (completedTaskCount >= taskList.size()) { + break; + } + } + } + + private Map<String, Long> getCount(String logType, String filter, String totalMetric) { String sql = String.format(Objects.requireNonNull(env.getProperty("JOB_LOG_COUNT")), totalMetric, logType, StrUtil.isBlankIfStr(filter) ? "" : "WHERE ".concat(filter)); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(sql).build()); if (!baseResult.isSuccess()) { log.error("Job-get log count failed, message: {}", baseResult.getMessage()); throw new QGWBusinessException(baseResult.getStatus(), baseResult.getCode(), "Job-get log count failed, message: " + baseResult.getMessage()); @@ -296,7 +775,7 @@ public class JobExecuteService implements EnvironmentAware { return map; } - private DecimalFormat getDecimalFormat() { + private static DecimalFormat getDecimalFormat() { DecimalFormat format = new DecimalFormat("#0.####"); format.applyPattern("0.0000"); format.setRoundingMode(RoundingMode.FLOOR); @@ -330,7 +809,7 @@ public class JobExecuteService implements EnvironmentAware { private PlainSelect getWhereOuter(String sql) throws JSQLParserException { Select parse = (Select) CCJSqlParserUtil.parse(sql); - return (PlainSelect) parse.getSelectBody(); + return (PlainSelect) parse.getSelectBody(); } private Interval parseInterval(List<Expression> list, Interval interval, String partitionKey) { @@ -382,7 +861,7 @@ public class JobExecuteService implements EnvironmentAware { } private long getCriticalValue(String str) { - return Long.parseLong(SQLFunctionUtil.getQueryValue("toUnixTimestamp(" + str + ")", DBTypeEnum.CLICKHOUSE.getValue())); + return Long.parseLong(SQLFunctionUtil.getQueryValue("toUnixTimestamp(" + str + ")", DBEngineType.CLICKHOUSE.getValue())); } @Data @@ -398,7 +877,7 @@ public class JobExecuteService implements EnvironmentAware { } @Data - class JobInfo { + class JobInfo implements Serializable { private String id; private long startTime = System.currentTimeMillis(); private long count = 0; @@ -414,8 +893,48 @@ public class JobExecuteService implements EnvironmentAware { } } - @Override + @Autowired public void setEnvironment(Environment environment) { this.env = environment; } + + @Autowired + public void setSqlSyncQueryService(SQLSyncQueryService sqlSyncQueryService) { + this.sqlSyncQueryService = sqlSyncQueryService; + } + + @Autowired + public void setTaskExecuteService(TaskExecuteService taskExecuteService) { + this.taskExecuteService = taskExecuteService; + } + + @Autowired + public void setDatabaseService(DatabaseService databaseService) { + this.databaseService = databaseService; + } + + @Autowired + public void setDslService(DSLService dslService) { + this.dslService = dslService; + } + + @Autowired + public void setPacketCombineDslService(PacketCombineDslService packetCombineDslService) { + this.packetCombineDslService = packetCombineDslService; + } + + @Autowired + public void setTrafficSpectrumDslService(TrafficSpectrumDslService trafficSpectrumDslService) { + this.trafficSpectrumDslService = trafficSpectrumDslService; + } + + @Autowired + public void setEngineConfigSource(EngineConfigSource engineConfigSource) { + this.engineConfigSource = engineConfigSource; + } + + @Autowired + public void setJobCfg(JobConfig jobCfg) { + this.jobCfg = jobCfg; + } } diff --git a/src/main/java/com/mesalab/services/service/impl/JobServiceImp.java b/src/main/java/com/mesalab/services/service/impl/JobServiceImpl.java index 6a2c0b7a..8f45d0f1 100644 --- a/src/main/java/com/mesalab/services/service/impl/JobServiceImp.java +++ b/src/main/java/com/mesalab/services/service/impl/JobServiceImpl.java @@ -1,6 +1,7 @@ package com.mesalab.services.service.impl; import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.date.DatePattern; import cn.hutool.core.util.BooleanUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.crypto.digest.DigestUtil; @@ -8,6 +9,7 @@ import cn.hutool.json.JSONUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.alibaba.fastjson2.JSON; +import com.geedgenetworks.utils.DateUtils; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.jfinal.plugin.activerecord.Db; @@ -19,15 +21,12 @@ import com.mesalab.common.exception.BusinessException; import com.mesalab.common.utils.SavedQueryResultUtils; import com.mesalab.common.utils.sqlparser.SQLFunctionUtil; import com.mesalab.common.utils.sqlparser.SQLHelper; -import com.mesalab.qgw.constant.QGWMessageConst; import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.qgw.model.basic.QueryProfile; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.SQLQueryContext; import com.mesalab.qgw.model.basic.HBaseAPISource; -import com.mesalab.qgw.service.QueryService; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.services.common.enums.MetricFunction; -import com.mesalab.services.common.enums.MetricType; +import com.mesalab.qgw.service.SQLSyncQueryService; +import com.mesalab.qgw.service.DatabaseService; import com.mesalab.services.configuration.JobConfig; import com.mesalab.services.service.JobService; import com.mesalab.services.common.property.SqlPropertySourceFactory; @@ -60,13 +59,13 @@ import java.util.stream.Collectors; */ @Service("jobService") @PropertySource(value = "classpath:job-sql-template.sql", factory = SqlPropertySourceFactory.class) -public class JobServiceImp implements JobService, EnvironmentAware { +public class JobServiceImpl implements JobService, EnvironmentAware { private static final Log log = LogFactory.get(); private static final String URL_PREFIX = "jdbc:phoenix:"; @Autowired - QueryService queryService; + SQLSyncQueryService sqlSyncQueryService; @Autowired - MetadataService metadataService; + DatabaseService databaseService; @Autowired JobExecuteService jobExecuteService; @Autowired @@ -103,24 +102,22 @@ public class JobServiceImp implements JobService, EnvironmentAware { @Override public BaseResult commitAdHocQuery(HashMap<String, Object> reqBody) { validateAdHocQuery(reqBody); - String id = DigestUtil.md5Hex16(String.valueOf(reqBody.get(JobConfig.KEY_QUERY_TYPE)).toLowerCase()) - .concat(JobConfig.DELIMIT) - .concat(DigestUtil.md5Hex16(String.valueOf(reqBody))); + String id = DigestUtil.md5Hex(String.valueOf(reqBody)); String queryType = String.valueOf(reqBody.get(JobConfig.KEY_QUERY_TYPE)); if (isExist(id)) { log.info("Hoc Job already exist, ID is {}, params is {}", id, JSON.toJSONString(reqBody)); - return BaseResultGenerator.successCreate(buildJobResource(id, queryType)); + return BaseResultGenerator.successCreate(buildJobResource(id)); } log.info("Add Hoc Job, ID is {}, params is {}", id, JSON.toJSONString(reqBody)); Map<String, Object> property = Maps.newLinkedHashMap(); if (JobConfig.FIELD_DISCOVERY.equals(queryType)) { initFieldDiscoveryJob(id, reqBody, property); - jobExecuteService.addExecutorFieldDiscovery(id, reqBody); + //jobExecuteService.addExecutorFieldDiscovery(id, reqBody); } else if (JobConfig.STATISTICS.equals(queryType)) { initStatisticsJob(id, reqBody, property); jobExecuteService.addExecutorStatistics(id, reqBody, property); } - return BaseResultGenerator.successCreate(buildJobResource(id, queryType)); + return BaseResultGenerator.successCreate(buildJobResource(id)); } private void initStatisticsJob(String id, HashMap<String, Object> reqBody, Map<String, Object> property) { @@ -151,39 +148,31 @@ public class JobServiceImp implements JobService, EnvironmentAware { validateReport(map); String sql = JSONUtil.quote(map.get(JobConfig.KEY_CUSTOM_SAVED_QUERY_SQL).toString(), false).replace("'", "\\'"); String id = getSameJob(sql); - String queryType = String.valueOf(map.get(JobConfig.KEY_QUERY_TYPE)); if (StringUtil.isNotBlank(id)) { - return BaseResultGenerator.successCreate(buildJobResource(id, queryType)); + return BaseResultGenerator.successCreate(buildJobResource(id)); } map.put("UUID", System.currentTimeMillis()); - id = DigestUtil.md5Hex16(JobConfig.SAVED_QUERY) - .concat(JobConfig.DELIMIT) - .concat(DigestUtil.md5Hex16(String.valueOf(map))); + id = DigestUtil.md5Hex(String.valueOf(map)); long currentTime = System.currentTimeMillis() / 1000; Db.update(String.format(Objects.requireNonNull(env.getProperty("SAVED_QUERY_JOB_INIT")), id, sql, currentTime, currentTime)); - return BaseResultGenerator.successCreate(buildJobResource(id, queryType)); + return BaseResultGenerator.successCreate(buildJobResource(id)); } @Override public BaseResult getSavedQueryStatus(String jobId) { - Map<String, Object> job = buildJobStatus(jobId); - return BaseResultGenerator.success("ok", Lists.newArrayList(job)); - } - - private Map<String, Object> buildJobStatus(String jobId) { - List<Object> query = Db.query(String.format(Objects.requireNonNull(env.getProperty("SAVED_QUERY_JOB_STATUS")), jobId, 1)); + List<Record> query = Db.find(String.format(Objects.requireNonNull(env.getProperty("SAVED_QUERY_JOB_STATUS")), jobId, 1)); if (CollectionUtil.isEmpty(query)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), "No job found with the given ID"); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), "Not Find Matching Job."); } - return buildJobStatus(Arrays.asList((Object[]) query.get(0))); + Map<String, Object> jobInfo = buildJobStatus(query.get(0).getColumns()); + return BaseResultGenerator.success(null, jobInfo, null, null, null); } @Override public BaseResult getSavedQueryResult(String jobId) { BaseResult statusBaseResult = getSavedQueryStatus(jobId); if (statusBaseResult.isSuccess()) { - List<HashMap<String, Object>> statusData = (List<HashMap<String, Object>>) statusBaseResult.getData(); - HashMap<String, Object> jobInfo = statusData.get(0); + Map<String, Object> jobInfo = (Map<String, Object>) statusBaseResult.getJob(); boolean isDone = Boolean.parseBoolean(jobInfo.get(JobConfig.IS_DONE).toString()); boolean isCanceled = Boolean.parseBoolean(jobInfo.get(JobConfig.IS_CANCELED).toString()); boolean isFailed = Boolean.parseBoolean(jobInfo.get(JobConfig.IS_FAILED).toString()); @@ -191,26 +180,26 @@ public class JobServiceImp implements JobService, EnvironmentAware { List<BaseResult> resultList = buildBaseResults(Lists.newArrayList(jobId), Lists.newArrayList(jobInfo)); return resultList.get(0); } else { - return BaseResultGenerator.success(null, jobInfo, null, null); + return BaseResultGenerator.success(null, jobInfo, statusBaseResult.getOutputMode(), null, null); } } - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), "Get job status error: " + statusBaseResult.getMessage())); + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Get job status error: " + statusBaseResult.getMessage())); } private List<BaseResult> buildBaseResults(List<String> ids, List<Map<String, Object>> jobStatus) { BaseResult<Object> resultBaseResult = SavedQueryResultUtils.getByRowKey(ids); if (!resultBaseResult.isSuccess()) { - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), "Get job result error: " + resultBaseResult.getMessage())); + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Get job result error: " + resultBaseResult.getMessage())); } List<Map<String, Object>> resultData = (List<Map<String, Object>>) resultBaseResult.getData(); List<BaseResult> resultList = Lists.newArrayList(); for (Map<String, Object> data : resultData) { BaseResult result; Map map = JSON.parseObject((String) data.get("result"), Map.class); - result = BaseResultGenerator.generate(Integer.parseInt(String.valueOf(map.get("status"))), ResultCodeEnum.SUCCESS.getCode(), String.valueOf(map.get("message")), - map.get("data"), map.get("meta"), (Map<String, Object>) (map.get(JobConfig.STATISTICS)), QueryFormatEnum.JSON.getValue()); + result = BaseResultGenerator.generate(Integer.parseInt(String.valueOf(map.get("status"))), ResultCodeEnum.SUCCESS.getCode(), true, String.valueOf(map.get("message")), + map.get("data"), map.get("meta"), (Map<String, Object>) (map.get(JobConfig.STATISTICS)), OutputMode.JSON.getValue()); String rowKey = String.valueOf(data.get("ROWKEY")); jobStatus.forEach(o -> { if (StrUtil.equals(rowKey, String.valueOf(o.get(JobConfig.JOB_ID)))) { @@ -225,23 +214,23 @@ public class JobServiceImp implements JobService, EnvironmentAware { @Override public BaseResult cancelSavedQuery(String jobId) { Db.update(String.format(Objects.requireNonNull(env.getProperty("SAVED_QUERY_JOB_UPDATE")), "is_valid", 0, jobId)); - return BaseResultGenerator.success(ResultStatusEnum.ACCEPTED.getCode(), "ok", null); + return BaseResultGenerator.success(HttpStatusCodeEnum.ACCEPTED.getCode(), "ok", null); } @Override public List<Map<String, Object>> batchSavedQueryStatus(List<String> ids) { - List<Object> data = Db.query(String.format(Objects.requireNonNull(env.getProperty("SAVED_QUERY_JOB_STATUS")), StrUtil.join("', '", ids), ids.size())); + List<Record> data = Db.find(String.format(Objects.requireNonNull(env.getProperty("SAVED_QUERY_JOB_STATUS")), StrUtil.join("', '", ids), ids.size())); List<Map<String, Object>> result = Lists.newArrayList(); data.forEach(x -> { - Map<String, Object> job = buildJobStatus(Arrays.asList((Object[]) x)); + Map<String, Object> job = buildJobStatus(x.getColumns()); result.add(job); }); if (result.size() != ids.size()) { List<String> resultIds = Lists.newArrayList(); result.forEach(o -> resultIds.add(String.valueOf(o.get(JobConfig.JOB_ID)))); List<String> collect = ids.stream().filter(item -> !resultIds.contains(item)).collect(Collectors.toList()); - throw new BusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Find Jobs: " + StrUtil.join(",", collect))); + throw new BusinessException(HttpStatusCodeEnum.NOT_FOUND.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), "Not Find Jobs: " + StrUtil.join(",", collect))); } return result; } @@ -272,8 +261,8 @@ public class JobServiceImp implements JobService, EnvironmentAware { } return resultList; } - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), "Get job status error: " + statusList)); + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "Get job status error: " + statusList)); } @Override @@ -286,17 +275,22 @@ public class JobServiceImp implements JobService, EnvironmentAware { return buildLongTermResult(id); } - - private Map<String, Object> buildJobStatus(List<Object> list) { + private Map<String, Object> buildJobStatus(Map<String, Object> data) { Map<String, Object> job = Maps.newLinkedHashMap(); - job.put(JobConfig.JOB_ID, list.get(8)); - job.put("is_done", "DONE".equalsIgnoreCase(String.valueOf(list.get(0)))); - job.put("done_progress", Math.min(Double.parseDouble(String.valueOf(list.get(1))), 1)); - job.put("is_canceled", !BooleanUtil.toBoolean(String.valueOf(list.get(3)))); - job.put("is_failed", BooleanUtil.toBoolean(String.valueOf(list.get(2)))); - job.put("start_time", list.get(4)); - job.put("end_time", list.get(5)); - job.put("count", list.get(6)); + job.put(JobConfig.JOB_ID, data.get("job_id")); + job.put(JobConfig.IS_DONE, "DONE".equalsIgnoreCase(String.valueOf(data.get("state")))); + job.put(JobConfig.DONE_PROGRESS, Math.min(Double.parseDouble(String.valueOf(data.get("done_progress"))), 1.0)); + job.put(JobConfig.IS_CANCELED, !BooleanUtil.toBoolean(String.valueOf(data.get("is_valid")))); + job.put(JobConfig.IS_FAILED, BooleanUtil.toBoolean(String.valueOf(data.get("is_failed")))); + job.put(JobConfig.REASON, data.get("result_message")); + job.put(JobConfig.START_TIME, Optional.ofNullable(data.get("start_time")).map(s -> DateUtils.convertTimestampToString(Long.parseLong(data.get("start_time").toString()), DatePattern.UTC_PATTERN)) + .orElse(null)); + job.put(JobConfig.END_TIME, Optional.ofNullable(data.get("end_time")).map(s -> DateUtils.convertTimestampToString(Long.parseLong(data.get("end_time").toString()), DatePattern.UTC_PATTERN)) + .orElse(null)); + Map<String, Object> links = Maps.newLinkedHashMap(); + links.put(JobConfig.LINKS_STATUS, "/v1/query/job/" + job.get(JobConfig.JOB_ID)); + links.put(JobConfig.LINKS_RESULT, "/v1/query/job/" + job.get(JobConfig.JOB_ID) + "/result"); + job.put(JobConfig.LINKS, links); return job; } @@ -337,7 +331,7 @@ public class JobServiceImp implements JobService, EnvironmentAware { job.put(JobConfig.DONE_PROGRESS, StringUtil.isEmpty(map.get(JobConfig.DONE_PROGRESS)) ? 0 : map.get(JobConfig.DONE_PROGRESS)); job.put(JobConfig.IS_CANCELED, StringUtil.isEmpty(map.get(JobConfig.IS_CANCELED)) ? false : map.get(JobConfig.IS_CANCELED)); job.put(JobConfig.IS_FAILED, StringUtil.isEmpty(map.get(JobConfig.IS_FAILED)) ? false : map.get(JobConfig.IS_FAILED)); - return BaseResultGenerator.success(statistics, job, meta, resultList); + return BaseResultGenerator.success(statistics, job, null, meta, resultList); } private BaseResult<List<Map<String, Object>>> buildResultFieldDiscovery(String id) { @@ -375,15 +369,15 @@ public class JobServiceImp implements JobService, EnvironmentAware { job.put(JobConfig.DONE_PROGRESS, StringUtil.isEmpty(map.get(JobConfig.DONE_PROGRESS)) ? 0 : map.get(JobConfig.DONE_PROGRESS)); job.put(JobConfig.IS_CANCELED, StringUtil.isEmpty(map.get(JobConfig.IS_CANCELED)) ? false : map.get(JobConfig.IS_CANCELED)); job.put(JobConfig.IS_FAILED, StringUtil.isEmpty(map.get(JobConfig.IS_FAILED)) ? false : map.get(JobConfig.IS_FAILED)); - return BaseResultGenerator.success(statistics, job, null, resultList); + return BaseResultGenerator.success(statistics, job, OutputMode.JSON.getValue(), null, resultList); } private boolean isExist(String id) { String sql = String.format(Objects.requireNonNull(env.getProperty("JOB_DETAIL")), id); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(sql).build()); if (!baseResult.isSuccess()) { log.error("Query job status error: {}", baseResult.getMessage()); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), "Query job status error: " + baseResult.getMessage()); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), "Query job status error: " + baseResult.getMessage()); } List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); Map<String, Object> map; @@ -403,9 +397,9 @@ public class JobServiceImp implements JobService, EnvironmentAware { property.put(JobConfig.JOB_PROPERTY_TYPE, JobConfig.FIELD_DISCOVERY); List<String> fields = (List<String>) body.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS); List<String> interim = Lists.newArrayList(fields); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(env.getProperty("JOB_GET_FIELD_DISCOVERY_ALL_COLUMN")).option(QueryOptionEnum.REAL_TIME.getValue()).build()); + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(env.getProperty("JOB_GET_FIELD_DISCOVERY_ALL_COLUMN")).option(QueryOption.REAL_TIME.getValue()).build()); if (!baseResult.isSuccess()) { - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), "Init job error: get field_discovery all column error: ".concat(baseResult.getMessage())); } List<Map<String, Object>> meta = (List<Map<String, Object>>) baseResult.getMeta(); @@ -526,8 +520,8 @@ public class JobServiceImp implements JobService, EnvironmentAware { property.put(JobConfig.JOB_PROPERTY_K_OPTION, JobConfig.JOB_PROPERTY_V_OPTION_TOP); } } catch (JSQLParserException | RuntimeException e) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "long_term job init error: ".concat(e.getMessage()))); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), "long_term job init error: ".concat(e.getMessage()))); } } @@ -569,32 +563,32 @@ public class JobServiceImp implements JobService, EnvironmentAware { private PlainSelect checkParam(String sql, net.sf.jsqlparser.statement.Statement statement) { SelectBody selectBody; if (!(statement instanceof Select) || !((selectBody = ((Select) statement).getSelectBody()) instanceof PlainSelect)) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), "long_term job init error, not support, SQL is: " + sql)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), "long_term job init error, not support, SQL is: " + sql)); } PlainSelect plainSelect = (PlainSelect) selectBody; PlainSelect plainSelectInner = SQLHelper.getSelectInner(sql); if (StringUtil.isEmpty(plainSelect.getGroupBy()) && StringUtil.isEmpty(plainSelectInner.getGroupBy())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), "long_term job init error, not support non-aggregate SQL is: " + sql)); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), "long_term job init error, not support non-aggregate SQL is: " + sql)); } return plainSelect; } private List<Map<String, Object>> validationData(String id) { - BaseResult baseResult = queryService.executeQuery( - QueryProfile.builder().option(QueryOptionEnum.REAL_TIME.getValue()) - .query(String.format(Objects.requireNonNull(env.getProperty("JOB_RESULT")), id)) + BaseResult baseResult = sqlSyncQueryService.executeQuery( + SQLQueryContext.builder().option(QueryOption.REAL_TIME.getValue()) + .originalSQL(String.format(Objects.requireNonNull(env.getProperty("JOB_RESULT")), id)) .build()); if (!baseResult.isSuccess()) { - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), - String.format(ResultCodeEnum.UNKNOWN_EXCEPTION.getMessage(), "get Job Result Error: ".concat(baseResult.getMessage()))); + throw new QGWBusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), + String.format(CommonErrorCode.UNKNOWN_EXCEPTION.getMessage(), "get Job Result Error: ".concat(baseResult.getMessage()))); } List<Map<String, Object>> data = (List<Map<String, Object>>) baseResult.getData(); if (StringUtil.isEmpty(data) || StringUtil.isEmpty(data.get(0).get(JobConfig.JOB_PROPERTY))) { - throw new QGWBusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), "Not Found This Job: ".concat(id))); + throw new QGWBusinessException(HttpStatusCodeEnum.NOT_FOUND.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), "Not Found This Job: ".concat(id))); } return data; } @@ -606,36 +600,24 @@ public class JobServiceImp implements JobService, EnvironmentAware { private void validateAdHocQuery(HashMap<String, Object> body) { Object type = body.get(JobConfig.KEY_QUERY_TYPE); if (JobConfig.FIELD_DISCOVERY.equals(type)) { - validCustomParamsOfFieldDiscovery(body); - List<String> fields = (List<String>) body.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS); - body.put(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS, fields.stream().distinct().collect(Collectors.toList())); + //validCustomParamsOfFieldDiscovery(body); + //List<String> fields = (List<String>) body.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS); + //body.put(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_FIELDS, fields.stream().distinct().collect(Collectors.toList())); syntaxValidationFieldDiscovery(body); } else if (JobConfig.STATISTICS.equals(type)) { syntaxValidationStatistics(body); } } - private void validCustomParamsOfFieldDiscovery(HashMap<String, Object> body) { - Object metric = body.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC); - if (StringUtil.isEmpty(metric)) { - return; - } - boolean validMetric = MetricType.isValid(String.valueOf(metric)); - boolean validFn = MetricFunction.isValid(String.valueOf(body.get(JobConfig.KEY_CUSTOM_FIELD_DISCOVERY_METRIC_FN))); - if (!validMetric || !validFn) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_CUSTOM_FIELD_DISCOVERY_ERROR)); - } - } private void validateReport(HashMap<String, Object> map) { - QueryProfile queryProfile = QueryProfile.builder().query(String.valueOf(map.get(JobConfig.KEY_CUSTOM_SAVED_QUERY_SQL))) - .option(QueryOptionEnum.SYNTAX_PARSE.getValue()) + SQLQueryContext queryProfile = SQLQueryContext.builder().originalSQL(String.valueOf(map.get(JobConfig.KEY_CUSTOM_SAVED_QUERY_SQL))) + .option(QueryOption.SYNTAX_PARSE.getValue()) .build(); - BaseResult baseResult = queryService.executeQuery(queryProfile); + BaseResult baseResult = sqlSyncQueryService.executeQuery(queryProfile); if (!baseResult.isSuccess()) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), baseResult.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), baseResult.getMessage())); } } @@ -652,20 +634,20 @@ public class JobServiceImp implements JobService, EnvironmentAware { List field = (List) map.get("custom.field_discovery.fields"); Object filter = map.get("custom.field_discovery.filter"); String validationSql = String.format(syntaxValidation, String.join(",", field), map.get("query.data_source"), StringUtil.isEmpty(filter) ? " 1 = 1" : filter); - QueryProfile queryProfile = QueryProfile.builder().query(validationSql).option(QueryOptionEnum.SYNTAX_VALIDATION.getValue()).build(); - BaseResult baseResult = queryService.executeQuery(queryProfile); + SQLQueryContext queryProfile = SQLQueryContext.builder().originalSQL(validationSql).option(QueryOption.SYNTAX_VALIDATION.getValue()).build(); + BaseResult baseResult = sqlSyncQueryService.executeQuery(queryProfile); if (!baseResult.isSuccess()) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), baseResult.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), baseResult.getMessage())); } } private void syntaxValidationStatistics(HashMap<String, Object> body) { String sql = String.valueOf(body.get(JobConfig.KEY_CUSTOM_STATISTICS_SQL)); - BaseResult baseResult = queryService.executeQuery(QueryProfile.builder().query(sql).option(QueryOptionEnum.SYNTAX_VALIDATION.getValue()).build()); + BaseResult baseResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(sql).option(QueryOption.SYNTAX_VALIDATION.getValue()).build()); if (!baseResult.isSuccess()) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), baseResult.getMessage())); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), baseResult.getMessage())); } } @@ -674,7 +656,7 @@ public class JobServiceImp implements JobService, EnvironmentAware { java.sql.Statement statement = null; int i; try { - log.info("{} engine execute update: {}", DBTypeEnum.HBASE.getValue(), sql); + log.info("{} engine execute update: {}", DBEngineType.HBASE.getValue(), sql); PROPERTIES.setProperty("hbase.rpc.timeout", hBaseAPISource.getRpcTimeout()); PROPERTIES.setProperty("hbase.client.scanner.timeout.period", hBaseAPISource.getRpcTimeout()); PROPERTIES.setProperty("phoenix.query.timeoutMs", hBaseAPISource.getRpcTimeout()); @@ -684,18 +666,17 @@ public class JobServiceImp implements JobService, EnvironmentAware { i = statement.executeUpdate(sql); conn.commit(); } catch (RuntimeException | SQLException e) { - log.error("{} engine error: {}", DBTypeEnum.HBASE.getValue(), e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), e.getMessage()); + log.error("{} engine error: {}", DBEngineType.HBASE.getValue(), e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), e.getMessage()); } finally { close(statement, conn); } return i == 1; } - private Map<String, Object> buildJobResource(String id, String queryType) { + private Map<String, Object> buildJobResource(String id) { Map<String, Object> result = Maps.newHashMap(); result.put(JobConfig.JOB_ID, id); - result.put(JobConfig.JOB_RESOURCE_PATH, id.concat("/").concat(queryType)); return result; } @@ -703,7 +684,7 @@ public class JobServiceImp implements JobService, EnvironmentAware { Connection conn = null; java.sql.Statement statement = null; try { - log.info("DB Engine is :{}, Execute is: {}", DBTypeEnum.HBASE.getValue(), sql); + log.info("DB Engine is :{}, Execute is: {}", DBEngineType.HBASE.getValue(), sql); PROPERTIES.setProperty("hbase.rpc.timeout", hBaseAPISource.getRpcTimeout()); PROPERTIES.setProperty("hbase.client.scanner.timeout.period", hBaseAPISource.getRpcTimeout()); PROPERTIES.setProperty("phoenix.query.timeoutMs", hBaseAPISource.getRpcTimeout()); @@ -713,9 +694,9 @@ public class JobServiceImp implements JobService, EnvironmentAware { statement.execute(sql); conn.commit(); } catch (RuntimeException | SQLException e) { - log.error("DB Engine is :{}, Query is error: {}", DBTypeEnum.HBASE.getValue(), e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); + log.error("DB Engine is :{}, Query is error: {}", DBEngineType.HBASE.getValue(), e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); } finally { diff --git a/src/main/java/com/mesalab/services/service/impl/KBServiceImp.java b/src/main/java/com/mesalab/services/service/impl/KBServiceImp.java new file mode 100644 index 00000000..465de54d --- /dev/null +++ b/src/main/java/com/mesalab/services/service/impl/KBServiceImp.java @@ -0,0 +1,362 @@ +package com.mesalab.services.service.impl; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.map.MapUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.core.util.XmlUtil; +import cn.hutool.crypto.digest.DigestUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.geedgenetworks.utils.DateUtils; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.entity.BaseResultGenerator; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.common.exception.BusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.common.utils.RandomUtil; +import com.mesalab.qgw.model.basic.HttpConfig; +import com.mesalab.qgw.model.basic.HttpResponseResult; +import com.mesalab.qgw.service.impl.HttpClientServiceV2; +import com.mesalab.services.common.entity.KnowledgeBaseRequest; +import com.mesalab.services.common.entity.KnowledgeConstant; +import com.mesalab.services.configuration.HosConfig; +import com.mesalab.services.service.KBService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.util.*; +import java.util.stream.Collectors; + +@Service("kbService") +public class KBServiceImp implements KBService { + private static final Log log = LogFactory.get(); + @Autowired + private HosConfig hosConfig; + @Autowired + private HttpClientServiceV2 httpClientServiceV2; + @Autowired + private HttpConfig httpConfig; + private static final String X_HOS_METADATA_PREFIX = "x-hos-meta-"; + private static final String INITIAL_VERSION = "1.0"; + private static final String NAME = "Name"; + private static final String CATEGORY = "Category"; + private static final String ORIGIN_URL = "OriginUrl"; + private static final String GENERATED_TIME = "GeneratedTime"; + private static final String LAST_UPDATE_TIME = "LastUpdateTime"; + private static final String FORMAT = "Format"; + private static final String SHA_256 = "Sha256"; + private static final String VERSION = "Version"; + private static final String IS_VALID = "IsValid"; + private static final String KEY = "Key"; + private static final String SIZE = "Size"; + private static final String USER_DEFINED_META = "UserDefinedMeta"; + private static final String KEY_COUNT = "KeyCount"; + private static final String CONTENTS = "Contents"; + private static final String KB_ID = "KbId"; + + @Override + public BaseResult publishKnowledge(MultipartFile file, KnowledgeBaseRequest request) { + List<Map<String, Object>> dataList; + if (StrUtil.isBlank(request.getKbId()) && StrUtil.isBlank(request.getVersion())) { + request.setKbId(RandomUtil.getUUID()); + request.setVersion(INITIAL_VERSION); + } else { + if (StrUtil.isBlankIfStr(request.getVersion())) { + request.setVersion(generateLatestVersion(request)); + } + } + List<String> modifiedVersionList = Lists.newArrayList(); + publishFile(file, request); + modifiedVersionList.add(request.getVersion()); + generateLatestFile(file, request); + modifiedVersionList.add(KnowledgeConstant.LATEST); + String getFileListUrl = hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("?").concat("prefix").concat("=").concat(request.getKbId()); + dataList = filterResult("version", modifiedVersionList, generateResult(getContentList(getFileListUrl))); + return BaseResultGenerator.success(dataList); + } + + + @Override + public BaseResult updateKnowledge(MultipartFile file, KnowledgeBaseRequest request) { + List<Map<String, Object>> dataList = Lists.newArrayList(); + Map<String, String> headers = buildHeaderMap(request); + publishFile(file, request); + dataList.add(generateSingleResult(request, headers)); + return BaseResultGenerator.success(dataList); + } + + @Override + public BaseResult deleteKnowledge(KnowledgeBaseRequest request) { + List<Map<String, Object>> dataList = Lists.newArrayList(); + if (StrUtil.isBlankIfStr(request.getVersion())) { + String getFileListUrl = hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("?").concat("prefix").concat("=").concat(request.getKbId()); + dataList = generateResult(getContentList(getFileListUrl)); + deleteFileById(request, getContentList(getFileListUrl)); + } else { + deleteFile(request); + dataList.add(generateSingleResult(request, Maps.newHashMap())); + } + return BaseResultGenerator.success(dataList); + } + + @Override + public BaseResult updateStatus(KnowledgeBaseRequest request) { + List<Map<String, Object>> dataList = Lists.newArrayList(); + if (StrUtil.isBlankIfStr(request.getVersion())) { + String getFileListUrl = hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("?").concat("prefix").concat("=").concat(request.getKbId()); + dataList = generateResult(getContentList(getFileListUrl)); + updateFileStatusById(request, getContentList(getFileListUrl)); + } else { + updateFileStatus(request, buildHeaderMap(request)); + dataList.add(generateSingleResult(request, buildHeaderMap(request))); + } + return BaseResultGenerator.success(dataList); + } + + + @Override + public BaseResult getList(String kbId, String category) { + List<Map<String, Object>> dataList; + StringBuilder getListUrl = new StringBuilder(); + if (StrUtil.isNotBlank(kbId)) { + getListUrl.append(hosConfig.getUri()).append("/").append(hosConfig.getBucket()).append("?prefix=").append(kbId).append("_"); + if (StrUtil.isNotBlank(category)) { + getListUrl.append("&").append(X_HOS_METADATA_PREFIX).append("category=").append(category); + } + dataList = filterResult("kb_id", Arrays.asList(kbId), generateFileListResult(getContentList(getListUrl.toString()))); + } else { + getListUrl.append(hosConfig.getUri()).append("/").append(hosConfig.getBucket()); + if (StrUtil.isNotBlank(category)) { + getListUrl.append("?").append(X_HOS_METADATA_PREFIX).append("category=").append(category); + } + dataList = generateFileListResult(getContentList(getListUrl.toString())); + } + return BaseResultGenerator.success(dataList); + } + + private void generateLatestFile(MultipartFile file, KnowledgeBaseRequest request) { + request.setVersion(KnowledgeConstant.LATEST); + String currentDate = DateUtils.getCurrentDate("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + request.setOriginUrl(hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("/").concat(request.getKbId().concat("_").concat("latest"))); + request.setGeneratedTime(StrUtil.isBlank(request.getGeneratedTime()) ? currentDate : request.getGeneratedTime()); + request.setLastUpdateTime(StrUtil.isBlank(request.getLastUpdateTime()) ? currentDate : request.getLastUpdateTime()); + publishFile(file, request); + } + + private String generateLatestVersion(KnowledgeBaseRequest request) { + String getLatestFileUrl = hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("?").concat("prefix").concat("=").concat(request.getKbId()); + List<Map<String, String>> contentList = getContentList(getLatestFileUrl); + int majorMax = 1; + int minorMax = 0; + if (CollectionUtil.isNotEmpty(contentList)) { + for (Map contents : contentList) { + Map<String, String> udfMeta = (Map<String, String>) contents.get(USER_DEFINED_META); + if (MapUtil.isNotEmpty(udfMeta)) { + if (!String.valueOf(udfMeta.get(VERSION)).equals(KnowledgeConstant.LATEST)) { + String[] versionParts = udfMeta.get(VERSION).split("\\."); + int major = Integer.parseInt(versionParts[0]); + int minor = Integer.parseInt(versionParts[1]); + majorMax = Math.max(majorMax, major); + minorMax = Math.max(minorMax, minor); + } + } + } + } + return incrementVersion(majorMax + "." + minorMax); + } + + private List<Map<String, String>> getContentList(String getLatestFileUrl) { + List<Map<String, String>> contentList = Lists.newArrayList(); + Map<String, String> headers = new HashMap<>(); + headers.put(KnowledgeConstant.TOKEN, hosConfig.getToken()); + HttpResponseResult httpResponseResult = httpClientServiceV2.get(getLatestFileUrl, 60000, headers); + Map<String, Object> resultMap = XmlUtil.xmlToMap(String.valueOf(httpResponseResult.getResponseBody())); + int count = Integer.parseInt(String.valueOf(resultMap.get(KEY_COUNT))); + switch (count) { + case 0: + return contentList; + case 1: + contentList.add((Map<String, String>) resultMap.get(CONTENTS)); + break; + default: + contentList = (List<Map<String, String>>) resultMap.get(CONTENTS); + } + return contentList; + } + + + private Map<String, String> buildHeaderMap(KnowledgeBaseRequest request) { + Map<String, String> headers = Maps.newHashMap(); + String currentDate = DateUtils.getCurrentDate("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + request.setOriginUrl(StrUtil.isBlank(request.getOriginUrl()) ? hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("/").concat(request.getKbId().concat("_").concat(request.getVersion())) : request.getOriginUrl()); + request.setGeneratedTime(StrUtil.isBlank(request.getGeneratedTime()) ? currentDate : request.getGeneratedTime()); + request.setLastUpdateTime(StrUtil.isBlank(request.getLastUpdateTime()) ? currentDate : request.getLastUpdateTime()); + headers.put(KnowledgeConstant.TOKEN, hosConfig.getToken()); + Field[] declaredFields = request.getClass().getDeclaredFields(); + for (Field field : declaredFields) { + field.setAccessible(true); + String fieldName = field.getName(); + if (!fieldName.equals("kb_id")) { + try { + Object o = field.get(request); + if (o != null) { + headers.put(X_HOS_METADATA_PREFIX.concat(StrUtil.toSymbolCase(fieldName, '-')), o.toString()); + } + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } + return headers; + } + + private void publishFile(MultipartFile file, KnowledgeBaseRequest request) { + String putFileUrl = hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("/").concat(request.getKbId().concat("_").concat(request.getVersion())); + try { + request.setSha256(DigestUtil.sha256Hex(file.getBytes())); + Map<String, String> headers = buildHeaderMap(request); + InputStream in = new ByteArrayInputStream(file.getBytes()); + HttpResponseResult put = httpClientServiceV2.put(putFileUrl, in, httpConfig.getServerResponseTimeOut(), headers); + if (HttpStatusCodeEnum.SUCCESS.getCode() != put.getStatusCode()) { + log.error("knowledge base file upload failed."); + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), "knowledge base file upload failed."); + } + } catch (IOException e) { + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), e.getMessage()); + } + } + + private void deleteFileById(KnowledgeBaseRequest request, List<Map<String, String>> contentList) { + for (Map contents : contentList) { + Map<String, String> udfMeta = (Map<String, String>) contents.get(USER_DEFINED_META); + request.setVersion(String.valueOf(udfMeta.get(VERSION))); + deleteFile(request); + } + } + + private void deleteFile(KnowledgeBaseRequest request) { + String fileName = request.getKbId().concat("_").concat(request.getVersion()); + String deleteFileUrl = hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("/").concat(fileName); + Map<String, String> headers = Maps.newHashMap(); + headers.put(KnowledgeConstant.TOKEN, hosConfig.getToken()); + try { + httpClientServiceV2.delete(deleteFileUrl, httpConfig.getServerRequestTimeOut(), headers); + } catch (RuntimeException e) { + log.error("knowledge base file delete error: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } + } + + private void updateFileStatusById(KnowledgeBaseRequest request, List<Map<String, String>> contentList) { + for (Map contents : contentList) { + Map<String, String> udfMeta = (Map<String, String>) contents.get(USER_DEFINED_META); + request.setVersion(String.valueOf(udfMeta.get(VERSION))); + updateFileStatus(request, buildHeaderMap(request)); + } + } + + private void updateFileStatus(KnowledgeBaseRequest request, Map<String, String> headers) { + String fileName = request.getKbId().concat("_").concat(request.getVersion()); + String updateStatusFileUrl = hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("/").concat(fileName).concat("?metadata"); + HttpResponseResult put = httpClientServiceV2.put(updateStatusFileUrl, null, httpConfig.getServerResponseTimeOut(), headers); + if (put.getStatusCode() != HttpStatusCodeEnum.SUCCESS.getCode()) { + log.error("knowledge base file update status failed."); + throw new BusinessException(HttpStatusCodeEnum.SERVER_ERROR.getCode(), CommonErrorCode.UNKNOWN_EXCEPTION.getCode(), "knowledge base file update status failed."); + } + } + + private List<Map<String, Object>> generateResult(List<Map<String, String>> contentList) { + List<Map<String, Object>> dataList = Lists.newArrayList(); + if (CollectionUtil.isNotEmpty(contentList)) { + for (Map contents : contentList) { + Map<String, Object> map1 = Maps.newHashMap(); + Map<String, String> udfMeta = (Map<String, String>) contents.get(USER_DEFINED_META); + map1.put("version", String.valueOf(udfMeta.get(VERSION))); + map1.put("url", String.valueOf(udfMeta.get(ORIGIN_URL))); + map1.put("generated_time", String.valueOf(udfMeta.get(GENERATED_TIME))); + map1.put("last_update_time", String.valueOf(udfMeta.get(LAST_UPDATE_TIME))); + map1.put("kb_id", String.valueOf(udfMeta.get(KB_ID))); + dataList.add(map1); + } + } + return dataList; + } + + private Map<String, Object> generateSingleResult(KnowledgeBaseRequest request, Map<String, String> headers) { + Map<String, Object> dataMap = Maps.newHashMap(); + String currentDate = DateUtils.getCurrentDate("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + dataMap.put("version", request.getVersion()); + dataMap.put("url", hosConfig.getUri().concat("/").concat(hosConfig.getBucket()).concat("/").concat(request.getKbId().concat("_").concat(request.getVersion()))); + dataMap.put("generated_time", StrUtil.isNotBlank(headers.get(X_HOS_METADATA_PREFIX.concat("generated-time"))) ? headers.get(X_HOS_METADATA_PREFIX.concat("generated-time")) : currentDate); + dataMap.put("last_update_time", StrUtil.isNotBlank(headers.get(X_HOS_METADATA_PREFIX.concat("last-update-time"))) ? headers.get(X_HOS_METADATA_PREFIX.concat("last-update-time")) : currentDate); + dataMap.put("kb_id", request.getKbId()); + return dataMap; + } + + private List<Map<String, Object>> generateFileListResult(List<Map<String, String>> contentList) { + List<Map<String, Object>> dataList = Lists.newArrayList(); + if (CollectionUtil.isNotEmpty(contentList)) { + for (Map contents : contentList) { + Map<String, Object> map1 = Maps.newHashMap(); + if (MapUtil.isNotEmpty((Map<?, ?>) contents.get(USER_DEFINED_META))) { + Map<String, String> udfMeta = (Map<String, String>) contents.get(USER_DEFINED_META); + map1.put("kb_id", StrUtil.isBlankIfStr(udfMeta.get(KB_ID)) ? "" : udfMeta.get(KB_ID)); + map1.put("name", StrUtil.isBlankIfStr(udfMeta.get(NAME)) ? "" : udfMeta.get(NAME)); + map1.put("category", StrUtil.isBlankIfStr(udfMeta.get(CATEGORY)) ? "" : udfMeta.get(CATEGORY)); + map1.put("path", StrUtil.isBlankIfStr(udfMeta.get(ORIGIN_URL)) ? "" : udfMeta.get(ORIGIN_URL)); + map1.put("generated_time", StrUtil.isBlankIfStr(udfMeta.get(GENERATED_TIME)) ? "" : udfMeta.get(GENERATED_TIME)); + map1.put("last_update_time", StrUtil.isBlankIfStr(udfMeta.get(LAST_UPDATE_TIME)) ? "" : udfMeta.get(LAST_UPDATE_TIME)); + map1.put("format", StrUtil.isBlankIfStr(udfMeta.get(FORMAT)) ? "" : udfMeta.get(FORMAT)); + map1.put("sha256", StrUtil.isBlankIfStr(udfMeta.get(SHA_256)) ? "" : udfMeta.get(SHA_256)); + map1.put("version", StrUtil.isBlankIfStr(udfMeta.get(VERSION)) ? "" : udfMeta.get(VERSION)); + map1.put("is_valid", StrUtil.isBlankIfStr(udfMeta.get(IS_VALID)) ? 1 : Integer.parseInt(udfMeta.get(IS_VALID))); + } else { + map1.put("name", ""); + map1.put("category", ""); + map1.put("path", ""); + map1.put("generated_time", ""); + map1.put("last_update_time", ""); + map1.put("format", ""); + map1.put("sha256", ""); + map1.put("version", ""); + map1.put("is_valid", 1); + map1.put("kb_id", StrUtil.subBefore(String.valueOf(contents.get(KEY)), "_", true)); + } + map1.put("size", StrUtil.isBlankIfStr(contents.get(SIZE)) ? 0 : Integer.parseInt(String.valueOf(contents.get(SIZE)))); + dataList.add(map1); + } + } + return dataList; + } + + private List<Map<String, Object>> filterResult(String filterKey, List<String> filterValues, List<Map<String, Object>> list) { + return list.stream() + .filter(map -> filterValues.contains(map.get(filterKey))) + .collect(Collectors.toList()); + } + + private String incrementVersion(String version) { + String[] parts = version.split("\\."); + int[] nums = new int[parts.length]; + for (int i = 0; i < parts.length; i++) { + nums[i] = Integer.parseInt(parts[i]); + } + + int idx = nums.length - 1; + nums[idx]++; + + StringBuilder sb = new StringBuilder(); + for (int num : nums) { + sb.append(num); + sb.append('.'); + } + sb.setLength(sb.length() - 1); + return sb.toString(); + } +} diff --git a/src/main/java/com/mesalab/services/service/impl/KnowledgeBaseServiceImpl.java b/src/main/java/com/mesalab/services/service/impl/KnowledgeBaseServiceImpl.java deleted file mode 100644 index 09a28305..00000000 --- a/src/main/java/com/mesalab/services/service/impl/KnowledgeBaseServiceImpl.java +++ /dev/null @@ -1,516 +0,0 @@ -package com.mesalab.services.service.impl; - -import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.io.FileUtil; -import cn.hutool.core.io.file.FileMode; -import cn.hutool.core.text.StrFormatter; -import cn.hutool.core.util.NumberUtil; -import cn.hutool.core.util.XmlUtil; -import cn.hutool.crypto.digest.DigestUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.alibaba.fastjson2.JSON; -import com.alibaba.nacos.api.config.ConfigService; -import com.alibaba.nacos.api.exception.NacosException; -import com.google.common.collect.Maps; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.nacos.NacosConst; -import com.mesalab.qgw.model.basic.HttpConfig; -import com.mesalab.qgw.service.impl.HttpClientService; -import com.mesalab.services.common.entity.KnowledgeBase; -import com.mesalab.services.common.entity.KnowledgeConstant; -import com.mesalab.services.configuration.HosConfig; -import com.mesalab.services.service.KnowledgeBaseService; -import com.geedgenetworks.utils.DateUtils; -import com.geedgenetworks.utils.StringUtil; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.http.Header; -import org.apache.http.HttpResponse; -import org.apache.http.message.BasicHeader; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import javax.xml.xpath.XPathConstants; -import java.io.*; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - - -@Service -public class KnowledgeBaseServiceImpl implements KnowledgeBaseService { - - private static final Log log = LogFactory.get(); - - @Autowired - private ConfigService pubConfigService; - - @Autowired - private HosConfig hosConfig; - - @Autowired - private HttpClientService httpClientService; - - @Autowired - private HttpConfig httpConfig; - - @Autowired - private CuratorFramework client; - - private static final String ZK_LOCK_PATH = "/zkLock"; - private static final String HOS_ETAG = "ETag"; - private static final String HOS_PART = "Part"; - private static final String HOS_PART_NUM = "PartNumber"; - private static final String HOS_COMPLETE_MULTIPART_UPLOAD = "CompleteMultipartUpload"; - - - @Override - public BaseResult queryKnowledge() { - List<KnowledgeBase> knowledgeBaseList = queryKnowledgeBaseList(); - return BaseResultGenerator.success(knowledgeBaseList); - } - - @Override - public BaseResult publishKnowledge(MultipartFile multipartFile, KnowledgeBase knowledge) throws Exception { - InterProcessMutex lock = new InterProcessMutex(client, ZK_LOCK_PATH); - try { - String id = null; - if (lock.acquire(10, TimeUnit.SECONDS)) { - List<KnowledgeBase> knowledgeBaseList = queryKnowledgeBaseList(); - String expirePath = buildKnowledgeBaseList(multipartFile, knowledge, knowledgeBaseList); - saveKnowledgeBaseList(knowledgeBaseList); - id = knowledge.getId(); - if (StringUtil.isNotEmpty(expirePath)) { - Header header = new BasicHeader(KnowledgeConstant.TOKEN, hosConfig.getToken()); - httpClientService.httpDelete(expirePath, httpConfig.getServerResponseTimeOut(), header); - } - } - Map<String, Object> map = Maps.newHashMap(); - map.put("id", id); - return BaseResultGenerator.success(map); - } catch (BusinessException e) { - log.error("publish knowledge error: {}", e.getMessage()); - throw new BusinessException(e.getStatus(), e.getCode(), e.getMessage()); - } finally { - lock.release(); - } - } - - @Override - public BaseResult updateKnowledge(MultipartFile file, KnowledgeBase knowledgeBase) throws Exception { - InterProcessMutex lock = new InterProcessMutex(client, ZK_LOCK_PATH); - try { - // 尝试获取锁，最长等待10s，超时放弃获取 - if (lock.acquire(10, TimeUnit.SECONDS)) { - List<KnowledgeBase> knowledgeBaseList = queryKnowledgeBaseList(); - String expirePath = null; - String newPath = null; - String fileSha256 = DigestUtil.sha256Hex(file.getBytes()); - String currentDate = DateUtils.getCurrentDate("yyyy-MM-dd'T'HH:mm:ssZ"); - for (KnowledgeBase knowledgeBaseRep : knowledgeBaseList) { - if (StringUtil.equals(knowledgeBase.getId(), knowledgeBaseRep.getId())) { - if (StringUtil.isBlank(newPath)) { - newPath = uploadFile(file); - expirePath = knowledgeBaseRep.getPath(); - } - if (StringUtil.isNotBlank(knowledgeBase.getName())) { - knowledgeBaseRep.setName(knowledgeBase.getName()); - } - if (StringUtil.isNotBlank(knowledgeBase.getFormat())) { - knowledgeBaseRep.setFormat(knowledgeBase.getFormat()); - } - if (StringUtil.isNotBlank(knowledgeBase.getType())) { - knowledgeBaseRep.setType(knowledgeBase.getType()); - } - if (StringUtil.isNotEmpty(knowledgeBase.getIsValid())) { - knowledgeBaseRep.setIsValid(knowledgeBase.getIsValid()); - } - if (StringUtil.isNotBlank(knowledgeBase.getOriginUrl())) { - knowledgeBaseRep.setOriginUrl(knowledgeBase.getOriginUrl()); - } - knowledgeBaseRep.setPath(newPath); - knowledgeBaseRep.setSha256(fileSha256); - knowledgeBaseRep.setSize(file.getSize()); - knowledgeBaseRep.setUpdateTime(currentDate); - - } - } - if (StringUtil.isNotBlank(newPath)) { - saveKnowledgeBaseList(knowledgeBaseList); - Header header = new BasicHeader(KnowledgeConstant.TOKEN, hosConfig.getToken()); - httpClientService.httpDelete(expirePath, httpConfig.getServerResponseTimeOut(), header); - } - } - } catch (RuntimeException e) { - log.error("update knowledge error: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } finally { - lock.release(); - } - Map<String, Object> map = Maps.newHashMap(); - map.put("id", knowledgeBase.getId()); - return BaseResultGenerator.success(map); - } - - @Override - public BaseResult deleteKnowledge(String id) throws Exception { - KnowledgeBase knowledgeBase = null; - InterProcessMutex lock = new InterProcessMutex(client, ZK_LOCK_PATH); - try { - if (lock.acquire(10, TimeUnit.SECONDS)) { - List<KnowledgeBase> knowledgeBaseList = queryKnowledgeBaseList(); - int originalSize = knowledgeBaseList.size(); - Iterator<KnowledgeBase> iterator = knowledgeBaseList.iterator(); - while (iterator.hasNext()) { - knowledgeBase = iterator.next(); - if (StringUtil.equals(id, knowledgeBase.getId())) { - iterator.remove(); - } - } - if (originalSize > knowledgeBaseList.size()) { - saveKnowledgeBaseList(knowledgeBaseList); - String path = knowledgeBase.getPath(); - Header header = new BasicHeader(KnowledgeConstant.TOKEN, hosConfig.getToken()); - httpClientService.httpDelete(path, httpConfig.getServerResponseTimeOut(), header); - } - } - } catch (RuntimeException e) { - log.error("delete knowledge error: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } finally { - lock.release(); - } - - - Map map = new HashMap(); - map.put("id", id); - return BaseResultGenerator.success(map); - } - - @Override - public BaseResult toggleKnowledge(KnowledgeBase knowledgeBase) throws Exception { - InterProcessMutex lock = new InterProcessMutex(client, ZK_LOCK_PATH); - try { - if (lock.acquire(10, TimeUnit.SECONDS)) { - List<KnowledgeBase> knowledgeBaseList = queryKnowledgeBaseList(); - for (int i = 0; i < knowledgeBaseList.size(); i++) { - KnowledgeBase knowledgeBaseRep = knowledgeBaseList.get(i); - if (StringUtil.equals(knowledgeBase.getId(), knowledgeBaseRep.getId())) { - knowledgeBaseRep.setIsValid(knowledgeBase.getIsValid()); - knowledgeBaseList.set(i,knowledgeBaseRep); - } - } - saveKnowledgeBaseList(knowledgeBaseList); - } - } catch (RuntimeException e) { - log.error("toggle knowledge error: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } finally { - lock.release(); - } - - Map<String, String> map = Maps.newHashMap(); - map.put("id", knowledgeBase.getId()); - return BaseResultGenerator.success(map); - } - - public List<KnowledgeBase> queryKnowledgeBaseList() { - List<KnowledgeBase> knowledgeBaseList = new ArrayList<>(); - try { - String config = pubConfigService.getConfig(NacosConst.KNOWLEDGE_BASE_DATA_ID, NacosConst.DEFAULT_GROUP, httpConfig.getServerRequestTimeOut()); - if (StringUtil.isNotEmpty(config)) { - knowledgeBaseList = JSON.parseArray(config, KnowledgeBase.class); - knowledgeBaseList = knowledgeBaseList.stream().sorted(Comparator.comparing(KnowledgeBase::getUpdateTime).reversed()).collect(Collectors.toList()); - } - } catch (NacosException e) { - log.error("get knowledge meta error: {}", e.getMessage()); - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(),e.getMessage()); - } - return knowledgeBaseList; - } - - - private String buildKnowledgeBaseList(MultipartFile file, KnowledgeBase knowledge, List<KnowledgeBase> knowledgeBaseList) throws Exception { - if (StringUtil.isEmpty(knowledge.getIsValid())) { - knowledge.setIsValid(KnowledgeConstant.VALID); - } - String newPath = uploadFile(file); - knowledge.setPath(newPath); - knowledge.setSha256(DigestUtil.sha256Hex(file.getBytes())); - knowledge.setSize(file.getSize()); - knowledge.setUpdateTime(DateUtils.getCurrentDate("yyyy-MM-dd'T'HH:mm:ssZ")); - - if (StringUtils.isNotBlank(knowledge.getId()) && knowledgeBaseList.stream().anyMatch(o -> StringUtil.equals(o.getId(), knowledge.getId()))) { - KnowledgeBase knowledgeRep = new KnowledgeBase(); - String expirePath = null; - for (KnowledgeBase item : knowledgeBaseList) { - if (StringUtil.equals(knowledge.getId(), item.getId())) { - expirePath = item.getPath(); - BeanUtils.copyProperties(item, knowledgeRep); - break; - } - } - for (KnowledgeBase item : knowledgeBaseList) { - if (StringUtil.equals(knowledge.getId(), item.getId())) { - knowledge.setVersion(item.getVersion()); - if (StringUtil.equals(item.getVersion(), KnowledgeConstant.LATEST)) { - BeanUtils.copyProperties(knowledge, item); - item.setVersion(KnowledgeConstant.LATEST); - } else { - BeanUtils.copyProperties(knowledge, item); - } - } - if (StringUtil.equals(item.getName(), knowledgeRep.getName()) && StringUtil.equals(item.getType(), knowledgeRep.getType()) && StringUtil.equals(item.getVersion(), KnowledgeConstant.LATEST)) { - BeanUtils.copyProperties(knowledge, item); - item.setVersion(KnowledgeConstant.LATEST); - } - } - return expirePath; - } else { - String lastVersion = queryLastKnowledgeVersion(knowledgeBaseList, knowledge); - String version = generateVersion(lastVersion); - if (StringUtil.isBlank(knowledge.getId())) { - knowledge.setId(DigestUtil.md5Hex16(knowledge.getType()).concat("-").concat(DigestUtil.md5Hex16(knowledge.getName().concat(knowledge.getFormat()).concat(version)))); - } - knowledge.setVersion(version); - knowledgeBaseList.add(knowledge); - addKnowledgeLatest(knowledge, knowledgeBaseList); - return null; - } - } - - private void addKnowledgeLatest(KnowledgeBase knowledge, List<KnowledgeBase> knowledgeBaseList) { - boolean hadLatestRep = knowledgeBaseList.stream().anyMatch(o -> StringUtil.equals(o.getName(), knowledge.getName()) - && StringUtil.equals(o.getType(), knowledge.getType()) - && StringUtil.equals(o.getVersion(), KnowledgeConstant.LATEST)); - if (hadLatestRep) { - for (KnowledgeBase item : knowledgeBaseList) { - if (!StringUtil.equals(item.getName(), knowledge.getName()) - || !StringUtil.equals(item.getType(), knowledge.getType()) - || !StringUtil.equals(item.getVersion(), KnowledgeConstant.LATEST)) { - continue; - } - BeanUtils.copyProperties(knowledge, item); - item.setVersion(KnowledgeConstant.LATEST); - } - } else { - KnowledgeBase knowledgeLatest = new KnowledgeBase(); - BeanUtils.copyProperties(knowledge, knowledgeLatest); - knowledgeLatest.setVersion(KnowledgeConstant.LATEST); - knowledgeBaseList.add(knowledgeLatest); - } - } - - private String queryLastKnowledgeVersion(List<KnowledgeBase> knowledgeBaseList, KnowledgeBase knowledge) { - if (CollectionUtil.isEmpty(knowledgeBaseList)) { - return null; - } - knowledgeBaseList = knowledgeBaseList.stream().sorted(Comparator.comparing(KnowledgeBase::getName) - .thenComparingInt(o -> { - String version = o.getVersion(); - int i = version.indexOf("_"); - String dateVersion = i > 0 ? version.substring(0, i) : "0"; - return Integer.parseInt( - NumberUtil.isInteger(dateVersion) ? dateVersion : "0"); - } - ) - .thenComparingInt(o -> { - String version = o.getVersion(); - int i = version.indexOf("_"); - String serial = i > 0 && i < version.length() - 1 ? version.substring(i + 1) : "0"; - return Integer.parseInt( - NumberUtil.isInteger(serial) ? serial : "0"); - } - ) - .reversed()).collect(Collectors.toList()); - for (KnowledgeBase knowledgeBase : knowledgeBaseList) { - if (!StringUtil.equals(knowledgeBase.getVersion(), KnowledgeConstant.LATEST) - && StringUtil.equals(knowledgeBase.getType(), knowledge.getType()) - && StringUtil.equals(knowledgeBase.getName(), knowledge.getName())) { - return knowledgeBase.getVersion(); - } - } - return null; - } - - private void saveKnowledgeBaseList(List<KnowledgeBase> knowledgeBaseList) throws BusinessException, InterruptedException { - - try { - boolean b = pubConfigService.publishConfig(NacosConst.KNOWLEDGE_BASE_DATA_ID, NacosConst.DEFAULT_GROUP, JSON.toJSONString(knowledgeBaseList)); - if (!b) { - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(),"update knowledge meta error."); - } - } catch (RuntimeException | NacosException e) { - log.error("update knowledge meta error: {}", e.getMessage()); - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(),e.getMessage()); - } - while (true) { - Thread.sleep(100); - String local = JSON.toJSONString(knowledgeBaseList.stream().sorted(Comparator.comparing(KnowledgeBase::getUpdateTime).reversed()).collect(Collectors.toList())); - String nacos = JSON.toJSONString(queryKnowledgeBaseList()); - if (StringUtil.equals(local, nacos)) { - break; - } - } - } - - - public String uploadFile(MultipartFile file) { - String uri = hosConfig.getUri(); - String bucket = hosConfig.getBucket(); - Header header = new BasicHeader(KnowledgeConstant.TOKEN, hosConfig.getToken()); - String[] split = file.getOriginalFilename().split("\\."); - String url = uri.concat("/").concat(bucket).concat("/").concat(UUID.randomUUID().toString()) - .concat("-").concat(Base64.getEncoder().encodeToString(split[0].getBytes())).concat(".").concat(split[1]); - try { - if (file.getBytes().length < hosConfig.getMultiUploadStartSize()) { - InputStream in = new ByteArrayInputStream(file.getBytes()); - HttpResponse httpResponse = httpClientService.httpPutStream(url, in, httpConfig.getServerResponseTimeOut(), header); - if (StringUtil.isEmpty(httpResponse) || httpResponse.getStatusLine().getStatusCode() != ResultStatusEnum.SUCCESS.getCode()) { - log.error("execute upload file failed."); - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(),"upload file failed."); - } - } else { - File fileTemp = null; - try { - fileTemp = new File(System.currentTimeMillis() + file.getOriginalFilename()); - String resBody = createMultipartUpload(url, header); - Document resXml = XmlUtil.parseXml(resBody); - Object uploadId = XmlUtil.getByXPath("//InitiateMultipartUploadResult/UploadId", resXml, XPathConstants.STRING); - Map<Integer, String> parts = Maps.newHashMap(); - FileUtils.copyInputStreamToFile(file.getInputStream(), fileTemp); - RandomAccessFile randomAccessFile = FileUtil.createRandomAccessFile(fileTemp, FileMode.r); - long totalLength = randomAccessFile.length(); - long hadReadLength = 0; - int count = 1; - while (hadReadLength < totalLength) { - byte[] waitUploadBytes = new byte[hosConfig.getMultiUploadPartSize() <= (totalLength - hadReadLength) ? hosConfig.getMultiUploadPartSize().intValue() : (int) (totalLength - hadReadLength)]; - hadReadLength = hadReadLength + randomAccessFile.read(waitUploadBytes); - InputStream inputStream = new ByteArrayInputStream(waitUploadBytes); - HttpResponse httpResponse = uploadPart(url, uploadId, count, inputStream, header); - Header[] eTags = httpResponse.getHeaders(HOS_ETAG); - parts.put(count++, eTags[0].getValue()); - } - Document completeBody = buildCompleteXmlReqBody(parts); - completeMultipartUpload(url, uploadId, completeBody, header); - } catch (IOException | RuntimeException ex) { - log.error("execute upload part file failed."); - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(),ex.getMessage()); - } finally { - if (fileTemp != null && fileTemp.exists()) { - FileUtil.del(fileTemp); - } - } - } - } catch (IOException e) { - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(),ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(),e.getMessage()); - } - return url; - } - - private String createMultipartUpload(String url, Header header) throws BusinessException { - String repStr = httpClientService.httpPost(url + "?uploads", null, header); - if ("-1".equals(repStr)) { - throw new BusinessException("create multipart upload failed"); - } - return repStr; - } - - private HttpResponse uploadPart(String url, Object uploadId, int times, InputStream inputStream, Header header) throws RuntimeException { - String urlParams = StrFormatter.format("?partNumber={}&uploadId={}", times, uploadId); - HttpResponse httpResponse = httpClientService.httpPutStream(url.concat(urlParams), inputStream, httpConfig.getServerResponseTimeOut(), header); - if (StringUtil.isEmpty(httpResponse) || httpResponse.getStatusLine().getStatusCode() != ResultStatusEnum.SUCCESS.getCode()) { - log.error("upload part file failed."); - throw new BusinessException("upload part file failed."); - } - return httpResponse; - } - - private void completeMultipartUpload(String url, Object uploadId, Document resXmlBody, Header header) throws RuntimeException { - String repStr = httpClientService.httpPost(url.concat(StrFormatter.format("?uploadId={}", uploadId)), XmlUtil.toStr(resXmlBody), header); - if ("-1".equals(repStr)) { - throw new BusinessException("complete multipart upload failed"); - } - } - - private Document buildCompleteXmlReqBody(Map<Integer, String> parts) { - Document xml = XmlUtil.createXml(); - xml.setXmlStandalone(true); - Element document = xml.createElement(HOS_COMPLETE_MULTIPART_UPLOAD); - for (Integer partNum : parts.keySet()) { - Element part = xml.createElement(HOS_PART); - Element eTag = xml.createElement(HOS_ETAG); - Element partNumber = xml.createElement(HOS_PART_NUM); - eTag.setTextContent(parts.get(partNum)); - partNumber.setTextContent(String.valueOf(partNum)); - part.appendChild(eTag); - part.appendChild(partNumber); - document.appendChild(part); - } - xml.appendChild(document); - return xml; - } - - private KnowledgeBase saveLatestKnowledge(KnowledgeBase knowledgeBase) { - KnowledgeBase base = new KnowledgeBase(); - BeanUtils.copyProperties(knowledgeBase, base); - base.setVersion(KnowledgeConstant.LATEST); - return base; - } - - private Boolean updateLatestKnowledge(List<KnowledgeBase> knowledgeBaseList, KnowledgeBase knowledgeBase) { - - String name = knowledgeBase.getName(); - String type = knowledgeBase.getType(); - - for (int i = 0; i < knowledgeBaseList.size(); i++) { - - KnowledgeBase knowledge = knowledgeBaseList.get(i); - if (StringUtil.equals(knowledge.getName(), name) && StringUtil.equals(knowledge.getType(), type) - && StringUtil.equals(knowledge.getVersion(), KnowledgeConstant.LATEST)) { - KnowledgeBase base = new KnowledgeBase(); - BeanUtils.copyProperties(knowledge, base); - base.setId(knowledgeBase.getId()); - base.setPath(knowledgeBase.getPath()); - base.setSize(knowledgeBase.getSize()); - base.setSha256(knowledgeBase.getSha256()); - base.setOriginUrl(knowledgeBase.getOriginUrl()); - base.setUpdateTime(knowledgeBase.getUpdateTime()); - base.setIsValid(knowledgeBase.getIsValid()); - knowledgeBaseList.set(i, base); - return false; - } - } - return true; - } - - public String generateVersion(String lastVersion) { - - String date = DateUtils.getCurrentDate("yyMMdd"); - if (StringUtils.isEmpty(lastVersion)) { - return date.concat("_1"); - } else { - String[] str = lastVersion.split("_"); - String index = str[1]; - if (StringUtil.equals(date, str[0])) { - return date.concat("_").concat(String.valueOf(Integer.parseInt(index) + 1)); - } else { - return date.concat("_1"); - } - } - - } - - -} diff --git a/src/main/java/com/mesalab/services/service/impl/RelationServiceImpl.java b/src/main/java/com/mesalab/services/service/impl/RelationServiceImpl.java deleted file mode 100644 index 4f37ff63..00000000 --- a/src/main/java/com/mesalab/services/service/impl/RelationServiceImpl.java +++ /dev/null @@ -1,326 +0,0 @@ -package com.mesalab.services.service.impl; - - -import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.util.ObjectUtil; -import cn.hutool.crypto.digest.DigestUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.alibaba.fastjson2.JSON; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.knowledge.enums.MatchEnum; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.model.basic.EntityConfigSource; -import com.mesalab.qgw.service.QueryService; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.common.dsl.ComDSLParse; -import com.mesalab.services.common.enums.EntityQueryType; -import com.mesalab.services.common.property.SqlPropertySourceFactory; -import com.mesalab.services.service.RelationService; -import com.geedgenetworks.utils.DateUtils; -import com.geedgenetworks.utils.StringUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.EnvironmentAware; -import org.springframework.context.annotation.PropertySource; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; - -import java.util.*; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.groupingBy; - -@Service -@PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) -public class RelationServiceImpl implements RelationService , EnvironmentAware { - - - @Value("${arango.maxrows}") - private String defLimit;//查询最大条数 - @Autowired - private QueryService queryService; - @Autowired - private EntityConfigSource entityConfigSource; - - private Environment env; - private static final Log log = LogFactory.get(); - - @Override - public BaseResult getRealRelation(String param, ComDSLObject comDSLObject) { - BaseResult baseResult; - if (param.equals(EntityQueryType.SUBSCRIBERIDPOOL.getType())) { - baseResult = formatResult(queryService.executeQuery(queryBuild(comDSLObject.getQuery()))); - } else if (param.equals(EntityQueryType.GTPC.getType())) { - String querySql = generateQuerySql(comDSLObject.getQuery()); - QueryProfile queryProfile = QueryProfile.builder().option("real_time").query(querySql).build(); - baseResult = deduplicationData(queryService.executeQuery(queryProfile),comDSLObject.getQuery()); - } else { - baseResult = BaseResultGenerator.failure(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), "No match queryType"); - } - if (!baseResult.isSuccess()) { - BaseResult result = JSON.parseObject(baseResult.getMessage(), BaseResult.class); - throw new QGWBusinessException(baseResult.getStatus(), baseResult.getCode(), result.getMessage()); - } - return baseResult; - } - - - private QueryProfile queryBuild(ComDSLObject.Query query) { - String range = ComDSLParse.parseRange(query.getParameters().getRange()); - List<ComDSLObject.Query.FilterBean> match = getMatchAfterTranslation(query.getParameters().getMatch()); - String fieldWhere = ComDSLParse.parseMath(match); - String timeWhere = ComDSLParse.parseInterval(query, "last_update_time", true); - String sql = String.format(Objects.requireNonNull(env.getProperty("RELATION_SUBSCRIBER_ID")), timeWhere.concat(range).concat(fieldWhere), ObjectUtil.isEmpty(query.getLimit()) ? defLimit : query.getLimit()); - log.info("real time relation subscribe id query sql: {}", sql); - return QueryProfile.builder().option("real_time").query(sql).build(); - } - - private List<ComDSLObject.Query.FilterBean> getMatchAfterTranslation(List<ComDSLObject.Query.FilterBean> match) { - if (ObjectUtil.isEmpty(match)) { - return Lists.newArrayList(); - } - for (ComDSLObject.Query.FilterBean filterBean : match) { - if (StringUtil.equals(MatchEnum.EXACTLY.getType(), filterBean.getType())) { - filterBean.setType(MatchEnum.PREFIX.getType()); - filterBean.setFieldKey("ROWKEY"); - List<String> fieldValues = filterBean.getFieldValues(); - fieldValues.replaceAll(s -> DigestUtil.md5Hex(s).concat("|")); - } else { - filterBean.setFieldKey("account"); - } - } - return match; - } - - private String generateQuerySql(ComDSLObject.Query query) { - - List<ComDSLObject.Query.FilterBean> match = query.getParameters().getMatch(); - StringBuffer fieldWhere = new StringBuffer(); - if (ObjectUtil.isNotEmpty(match)){ - StringBuffer filter = new StringBuffer(); - for (ComDSLObject.Query.FilterBean filterBean : match) { - match = new ArrayList<>(); - if (StringUtil.equals(MatchEnum.REGEX.getType(), filterBean.getType())) { - Map<String, ComDSLObject.Query.FilterBean> map = transformData(filterBean); - for (Map.Entry<String, ComDSLObject.Query.FilterBean> entry : map.entrySet()) { - ComDSLObject.Query.FilterBean bean = entry.getValue(); - match.add(generateRowKey(bean)); - } - filter.append(ComDSLParse.parseMath(match)); - } else { - match.add(generateRowKey(filterBean)); - filter.append(ComDSLParse.parseMath(match)); - } - - } - StringBuffer sb = new StringBuffer(" AND ("); - List<String> list = match.stream().filter(e -> !StringUtil.equals(e.getFieldKey(), "ROWKEY")).map(e -> e.getFieldKey()).distinct().collect(Collectors.toList()); - for (String str : list) { - if (StringUtil.equals(str,"imei")){ - sb.append("ROWKEY like '0%'"); - } else if (StringUtil.equals(str,"imsi")){ - sb.append("ROWKEY like '1%'"); - } else if (StringUtil.equals(str,"phone_number")){ - sb.append("ROWKEY like '2%'"); - } else if (StringUtil.equals(str,"apn")){ - sb.append("ROWKEY like '3%'"); - } - sb.append(" OR "); - } - - if (CollectionUtil.isNotEmpty(list)) { - fieldWhere = sb.delete(sb.length() - 3, sb.length()).append(")").append(filter); - } else { - fieldWhere = filter; - } - } - String range = ComDSLParse.parseRange(query.getParameters().getRange()); - - String sql = String.format(Objects.requireNonNull(env.getProperty("GTPC_KNOWLEDGE_BASE")), range.concat(fieldWhere.toString()), entityConfigSource.getTopGTPCDefaultSize()); - log.info("real time relation gtpc query sql: {}", sql); - return sql; - } - - private ComDSLObject.Query.FilterBean generateRowKey(ComDSLObject.Query.FilterBean filterBean) { - - - //Phone Number,APN 倒排 - String fieldKey = filterBean.getFieldKey(); - List<String> fieldValues = filterBean.getFieldValues(); - String type = filterBean.getType(); - //imei,imsi前缀匹配查询时，应使用Row Key进行筛选 - if (StringUtil.equals(type, MatchEnum.PREFIX.getType()) - && (StringUtil.equals(fieldKey, "imei") || StringUtil.equals(fieldKey, "imsi"))) { - - if (StringUtil.equals(fieldKey, "imei")) { - for (int i = 0; i < fieldValues.size(); i++) { - fieldValues.set(i,"0".concat(fieldValues.get(i))); - } - //phone_number,apn 后缀匹配查询时，应使用Row Key进行筛选 - } else if (StringUtil.equals(fieldKey, "imsi")) { - for (int i = 0; i < fieldValues.size(); i++) { - fieldValues.set(i,"1".concat(fieldValues.get(i))); - } - //phone_number,apn 后缀匹配查询时，应使用Row Key进行筛选 - } - filterBean.setFieldKey("ROWKEY"); - //phone_number,apn 后缀匹配查询时，应使用Row Key进行筛选 - } else if (StringUtil.equals(type, MatchEnum.SUFFIX.getType()) - && (StringUtil.equals(fieldKey, "phone_number") || StringUtil.equals(fieldKey, "apn"))) { - for (int i = 0; i < fieldValues.size(); i++) { - fieldValues.set(i, new StringBuffer(fieldValues.get(i)).reverse().toString()); - } - if (StringUtil.equals(fieldKey, "phone_number")) { - for (int i = 0; i < fieldValues.size(); i++) { - fieldValues.set(i,"2".concat(fieldValues.get(i))); - } - //phone_number,apn 后缀匹配查询时，应使用Row Key进行筛选 - } else if (StringUtil.equals(fieldKey, "apn")) { - for (int i = 0; i < fieldValues.size(); i++) { - fieldValues.set(i,"3".concat(fieldValues.get(i))); - } - //phone_number,apn 后缀匹配查询时，应使用Row Key进行筛选 - } - filterBean.setFieldKey("ROWKEY"); - filterBean.setType(MatchEnum.PREFIX.getType()); - } - - return filterBean; - } - - private Map<String, ComDSLObject.Query.FilterBean> transformData(ComDSLObject.Query.FilterBean filterBean) { - - Map<String, ComDSLObject.Query.FilterBean> map = new HashMap<>(); - List<String> fieldValues = filterBean.getFieldValues(); - String fieldKey = filterBean.getFieldKey(); - for (String value : fieldValues) { - String type = getTypeByValue(value); - String val = value.replaceAll("\\$", "").replaceAll("\\*", ""); - ComDSLObject.Query.FilterBean filter = map.get(type); - if (ObjectUtil.isEmpty(filter)) { - filter = new ComDSLObject.Query.FilterBean(); - List<String> fieldValueList = new ArrayList<>(); - filter.setType(type); - filter.setFieldKey(fieldKey); - fieldValueList.add(val); - filter.setFieldValues(fieldValueList); - map.put(type, filter); - } else { - filter.getFieldValues().add(val); - } - } - return map; - } - - private String getTypeByValue(String value) { - - String type; - if (value.startsWith("$")) { - type = MatchEnum.EXACTLY.getType(); - } else if (value.startsWith("*") && !value.endsWith("*")) { - type = MatchEnum.SUFFIX.getType(); - } else if (value.endsWith("*") && !value.startsWith("*")) { - type = MatchEnum.PREFIX.getType(); - } else if (value.startsWith("*") && value.endsWith("*")) { - type = MatchEnum.SUBSTRING.getType(); - } else { - type = MatchEnum.SUBSTRING.getType(); - } - return type; - - } - - /** - * 排序 - * @param mapList - * @param key - * @return - */ - private List<Map<String,Object>> sortByKey(List<Map<String,Object>> mapList ,String key){ - Collections.sort(mapList, new Comparator<Map<String, Object>>() { - @Override - public int compare(Map<String, Object> o1, Map<String, Object> o2) { - String name1 = o1.get(key).toString(); - String name2 = o2.get(key).toString(); - return name2.compareTo(name1); - } - }); - return mapList; - } - - - private BaseResult formatResult(BaseResult baseResult) { - - if (String.valueOf(ResultStatusEnum.SUCCESS.getCode()).equals(String.valueOf(baseResult.getStatus()))) { - baseResult.setMeta(null); - List<Map<String,Object>> resultList = new ObjectMapper().convertValue(baseResult.getData(), new TypeReference<List<Map<String,Object>>>() { - }); - LinkedHashMap<Object, List<Map<String,Object>>> listLinkedHashMap = resultList.stream() - .collect(groupingBy(e -> e.get("VSYS_ID") + "_" + e.get("IP"), LinkedHashMap::new, Collectors.toList())); - List<Map<String,Object>> list = new ArrayList<>(); - for (Map.Entry<Object, List<Map<String,Object>>> entry : listLinkedHashMap.entrySet()) { - List<Map<String, Object>> mapList = sortByKey(entry.getValue(), "LAST_FOUND_TIME"); - Map<String, Object> map = mapList.get(0); - map.put("FIRST_FOUND_TIME", DateUtils.convertTimestampToString(Long.parseLong(String.valueOf(map.get("FIRST_FOUND_TIME"))), DateUtils.YYYY_MM_DD_HH24_MM_SS)); - map.put("LAST_FOUND_TIME", DateUtils.convertTimestampToString(Long.parseLong(String.valueOf(map.get("LAST_FOUND_TIME"))), DateUtils.YYYY_MM_DD_HH24_MM_SS)); - list.add(map); - } - - return BaseResultGenerator.success(list); - } else { - baseResult = BaseResultGenerator.error(baseResult.getStatus(),baseResult.getMessage()); - } - return baseResult; - } - - private BaseResult deduplicationData(BaseResult baseResult, ComDSLObject.Query query) { - Object data = baseResult.getData(); - if (ObjectUtil.isNotEmpty(data)) { - List<Map<String, Object>> mapList = (List<Map<String, Object>>) data; - List<Map<String, Object>> deduplicationPhoneList = removeRepeatMapByKey(mapList, "phone_number"); - List<Map<String, Object>> deduplicationIMSIList = removeRepeatMapByKey(deduplicationPhoneList, "imsi"); - List<Map<String, Object>> list = removeRepeatMapByKey(deduplicationIMSIList, "teid"); - String limit = query.getLimit(); - if (StringUtil.isNotEmpty(limit) && Long.parseLong(limit) < list.size()) { - baseResult.setData(list.subList(0, Integer.parseInt(query.getLimit()))); - } else { - baseResult.setData(list); - } - } - return baseResult; - } - - /** - * 根据map中的某个key 去除List中重复的map - * @param resultList - * @param mapKey - * @return list - */ - - public List<Map<String, Object>> removeRepeatMapByKey(List<Map<String, Object>> resultList, String mapKey) { - - LinkedHashMap<Object, List<Map<String,Object>>> listLinkedHashMap = resultList.stream() - .collect(groupingBy(e -> e.get("vsys_id") + "_" + e.get(mapKey), LinkedHashMap::new, Collectors.toList())); - List<Map<String,Object>> list = new ArrayList<>(); - for (Map.Entry<Object, List<Map<String,Object>>> entry : listLinkedHashMap.entrySet()) { - List<Map<String, Object>> mapList = sortByKey(entry.getValue(), "last_update_time"); - Map<String, Object> map = mapList.get(0); - list.add(map); - } - return list; - - } - - @Override - public void setEnvironment(Environment environment) { - this.env = environment; - } -} diff --git a/src/main/java/com/mesalab/services/service/impl/SQLDatasetServiceImp.java b/src/main/java/com/mesalab/services/service/impl/SQLDatasetServiceImp.java deleted file mode 100644 index 475e4b83..00000000 --- a/src/main/java/com/mesalab/services/service/impl/SQLDatasetServiceImp.java +++ /dev/null @@ -1,421 +0,0 @@ -package com.mesalab.services.service.impl; - -import cn.hutool.core.util.StrUtil; -import cn.hutool.core.util.URLUtil; -import cn.hutool.crypto.digest.DigestUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.utils.Encodes; -import com.geedgenetworks.utils.StringUtil; -import com.google.common.base.CaseFormat; -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.jayway.jsonpath.JsonPath; -import com.jfinal.plugin.activerecord.Db; -import com.jfinal.plugin.activerecord.Record; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.*; -import com.mesalab.common.nacos.NacosConst; -import com.mesalab.common.utils.sqlparser.SQLHelper; -import com.mesalab.common.utils.sqlparser.SQLVisitorUtil; -import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.qgw.dialect.Dialect; -import com.mesalab.qgw.dialect.HbaseDialect; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.exception.QGWErrorCode; -import com.mesalab.qgw.model.basic.*; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.qgw.service.QueryService; -import com.mesalab.qgw.service.impl.HttpClientService; -import com.mesalab.services.common.property.SqlPropertySourceFactory; -import com.mesalab.services.service.SQLDatasetService; -import lombok.Data; -import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.select.Select; -import org.apache.commons.collections.CollectionUtils; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; -import org.apache.http.util.EntityUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.PropertySource; -import org.springframework.core.env.Environment; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.nio.charset.Charset; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * TODO - * - * @Classname SQLDatasetServiceImp - * @Date 2023/8/10 13:54 - * @Author wWei - */ -@Service("SQLDatasetService") -@PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) -public class SQLDatasetServiceImp - implements SQLDatasetService -{ - private static final Log log = LogFactory.get(); - private static Pattern pFieldVariable = Pattern.compile("\\$\\{(metric|dimension)_(.*?)\\}", Pattern.CASE_INSENSITIVE); - private static Pattern pLeftRightFlag = Pattern.compile("\\[\\[(.*?)\\]\\]", Pattern.CASE_INSENSITIVE); - private final static String TEMPLATE = "template"; - private final static String EXECUTE_ENGINE = "execute_engine"; - - public final static String queryIdSeparator = ":"; - - @Autowired - Environment env; - @Autowired - MetadataService metadataService; - @Autowired - QueryService queryService; - @Autowired - private HttpClientService httpClientService; - @Autowired - private HttpConfig httpConfig; - @Autowired - private ClickHouseHttpSource clickHouseHttpSource; - @Autowired - private DruidIoHttpSource druidIoHttpSource; - - @Override - public BaseResult getProcessesByQueryId(String queryId) { - Dialect dialect = buildAndGetDialect(queryId); - return dialect.getProcesses(queryId); - } - - @Override - public BaseResult deleteQueryTask(String queryId) { - Dialect dialect = buildAndGetDialect(queryId); - return dialect.executeKillQuery(queryId); - } - - @Override - public String getCustomQueryId(String resultId, String query) { - String tableName = SQLHelper.getTableName(query).get(0); - String dbType = metadataService.getDBTypeByTableName(tableName); - return DigestUtil.md5Hex(dbType) + queryIdSeparator + DigestUtil.md5Hex(resultId + query.trim()); - } - - @Override - public List<LinkedHashMap> getVariable() - { - Object codeInfo = metadataService.getCfg(NacosConst.SQL_DATASETS_VARIABLES); - if (StringUtil.isNotEmpty(codeInfo)) { - Object json = JSON.toJSON(codeInfo); - return JSON.parseArray(json.toString(), LinkedHashMap.class); - } - return Lists.newArrayList(); - } - - @Override - public Map<String, Object> getDataset(String datasetId) - { - List<Map<String, Object>> results = new ArrayList<>(); - List<Record> list = Db.find(buildGetDatasetSQL(Lists.newArrayList(datasetId), null, null)); - if (CollectionUtils.isEmpty(list)) { - return Maps.newHashMap(); - } - list.forEach(record -> results.add(record.getColumns())); - Map<String, Object> result = results.get(0); - log.info("SQL Dataset, ID is: {}, SQL is: {}", datasetId, result); - return result; - } - - @Override - public Map<String, Object> getDatasets(List<String> ids, String category, String title) - { - String sql = buildGetDatasetSQL(ids, category, title); - List<Map<String, Object>> data = new ArrayList<>(); - List<Record> list = Db.find(sql); - if (CollectionUtils.isEmpty(list)) { - return Maps.newHashMap(); - } - list.forEach(record -> data.add(record.getColumns())); - Map<String, Object> result = Maps.newHashMap(); - result.put("list", data); - return result; - } - - @Override - public BaseResult getPreview(String datasetId, String option) { - Map<String, Object> dataset = getDataset(datasetId); - if (dataset.isEmpty() || StrUtil.isBlankIfStr(dataset.get(TEMPLATE))) { - return BaseResultGenerator.success(); - } - String sqlTemplate = String.valueOf(dataset.get(TEMPLATE)); - String execute_engine = String.valueOf(dataset.get(EXECUTE_ENGINE)); - String sql = buildExecSQL(getVariable(), sqlTemplate); - log.info("Dataset Preview, ID is: {}, Exec SQL is: {}", datasetId, sql); - if (DiagnosisOptionEnum.EXECUTION.getValue().equalsIgnoreCase(option) || StrUtil.isBlank(option)) { - if (DBTypeEnum.CLICKHOUSE.getValue().equalsIgnoreCase(execute_engine)) { - return generateClickHouseBaseResult(sql); - } - if (DBTypeEnum.DRUID.getValue().equalsIgnoreCase(execute_engine)) { - return generateDruidBaseResult(sql); - } - if (DBTypeEnum.HBASE.getValue().equalsIgnoreCase(execute_engine)) { - return generateHbaseBaseResult(sql); - } - return getBaseResultByEngine(QueryProfile.builder().option(QueryOptionEnum.REAL_TIME.getValue()).query(sql).build()); - } else if (DiagnosisOptionEnum.PARSE.getValue().equalsIgnoreCase(option)) { - return getBaseResultByEngine(QueryProfile.builder().option(QueryOptionEnum.SYNTAX_PARSE.getValue()).query(sql).build()); - } else if (DiagnosisOptionEnum.VALIDATION.getValue().equalsIgnoreCase(option)) { - return getBaseResultByEngine(QueryProfile.builder().option(QueryOptionEnum.SYNTAX_VALIDATION.getValue()).query(sql).build()); - } else { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), - ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.OPTION_IS_NOT_SUPPORTED)); - } - } - - - @Override - public String buildExecSQL(List<LinkedHashMap> variables, String sql) - { - Matcher matcher = pLeftRightFlag.matcher(sql); - while (matcher.find()) { - sql = processOptionalClause(sql, "[[", "]]"); - } - sql = processFieldVariable(sql); - for (LinkedHashMap linkedHashMap : variables) { - String variable = String.valueOf(linkedHashMap.get("key")); - String def = String.valueOf(linkedHashMap.get("default")); - if ("filter".equals(variable) && StringUtil.isBlank(def)) { - def = " 1 = 1"; - } - String parameter = "${".concat(variable).concat("}"); - sql = sql.replace(parameter, def); - } - return sql; - } - - public Dialect buildAndGetDialect(String queryId) { - QueryProfile queryProfile = new QueryProfile(); - queryProfile.setDbType(parseDBType(queryId)); - return getDialect(queryProfile); - } - - private String parseDBType(String queryId) { - for (DBTypeEnum dbTypeEnum : DBTypeEnum.values()) { - if (queryId.startsWith(DigestUtil.md5Hex(dbTypeEnum.getValue()).concat(queryIdSeparator))) { - return dbTypeEnum.getValue(); - } - } - if (queryId.contains(queryIdSeparator)) { - throw new QGWBusinessException(ResultStatusEnum.NOT_FOUND.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(),QGWMessageConst.QUERY_ID_IS_NOT_SUPPORTED)); - } - return DBTypeEnum.CLICKHOUSE.getValue(); - } - - private Dialect getDialect(QueryProfile queryProfile) { - Dialect dialect; - try { - Class dialectClazz = Class.forName("com.mesalab.qgw.dialect." + - CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, queryProfile.getDbType()) + "Dialect"); - Constructor constructor = dialectClazz.getConstructor(QueryProfile.class); - dialect = (Dialect) constructor.newInstance(queryProfile); - } catch (ReflectiveOperationException | RuntimeException e) { - log.error("Dialect conversion instance exception:{}", e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),e.getMessage())); - } - return dialect; - } - - private String buildGetDatasetSQL(List<String> ids, String category, String title) - { - List<String> filterList = Lists.newArrayList(); - if (!ids.isEmpty()) { - filterList.add(" identifier_name IN ( '".concat(String.join("', '", ids)).concat("')")); - } - if (StrUtil.isNotBlank(category)) { - filterList.add(" category = '".concat(category.replace("'", "\\'")).concat("'")); - } - if (StrUtil.isNotBlank(title)) { - filterList.add(" title = '".concat(title.replace("'", "\\'")).concat("'")); - } - String filter = String.join(" AND ", filterList); - return String.format(Objects.requireNonNull(env.getProperty("SQL_DATASETS")), StrUtil.isNotBlank(filter) ? "WHERE ".concat(filter) : ""); - } - - private BaseResult getBaseResultByEngine(QueryProfile queryProfile) { - return queryService.executeQuery(queryProfile); - } - - private BaseResult generateDruidBaseResult(String sql) { - BaseResult baseResult; - Object dataObject; - Stopwatch watch = Stopwatch.createUnstarted(); - watch.start(); - Map<String, String> results = druidExecuteHttpPost(sql); - if (Integer.parseInt(results.get("status")) == ResultStatusEnum.SUCCESS.getCode()) { - baseResult = BaseResultGenerator.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.SUCCESS.getCode(), "ok", - (List<Object>) JSON.parseObject(results.get("result"), Object.class), null, null, QueryFormatEnum.JSON.getValue()); - dataObject = baseResult.getData(); - List<Object> allResults = (List<Object>) dataObject; - Map<String, Object> statistics = Maps.newLinkedHashMap(); - statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); - statistics.put("rows_read", allResults.size()); - baseResult.setData(allResults); - baseResult.setStatistics(statistics); - } else { - baseResult = BaseResultGenerator.generate(Integer.parseInt(results.get("status")), QGWErrorCode.SQL_BUILDER_EXCEPTION.getCode(), results.get("message"), - results.get("result"), null, null, QueryFormatEnum.JSON.getValue()); - dataObject = baseResult.getData(); - baseResult.setMessage("SQL Execution Error: ".concat(JsonPath.read(dataObject.toString(), "$.errorMessage"))); - } - return baseResult; - } - - - private Map<String, String> druidExecuteHttpPost(String sql) { - String queryURL = URLUtil.normalize(druidIoHttpSource.getUrl()); - DruidQueryParam druidQueryParam = new DruidQueryParam(); - druidQueryParam.setQuery(sql); - druidQueryParam.getContext().put("skipEmptyBuckets", druidIoHttpSource.getSkipEmptyBuckets()); - druidQueryParam.setResultFormat("object"); - log.info("DB Engine is :{}, Execute Query is: {}", DBTypeEnum.DRUID.getValue(), JSON.toJSONString(druidQueryParam)); - int socketTimeOut = httpConfig.getDruidSocketTimeOut(); - return httpClientService.httpPost(queryURL, JSON.toJSONString(druidQueryParam), socketTimeOut); - } - - - /** - * @param sql - * @return com.mesalab.common.base.BaseResult - * @Description ck SQL测试集在ClickHouse数据源查询并获取查询结果 - * @author wanghao - * @date 2021/9/9 17:39 - */ - private BaseResult generateClickHouseBaseResult(String sql) { - BaseResult baseResult; - Object dataObject; - Map<String, String> results = clickHouseExecuteHttpPost(sql); - if (Integer.parseInt(results.get("status")) == ResultStatusEnum.SUCCESS.getCode()) { - baseResult = BaseResultGenerator.generate(Integer.parseInt(results.get("status")), ResultCodeEnum.SUCCESS.getCode(), "ok", - JSON.parseObject(results.get("result"), Map.class), null, null, QueryFormatEnum.JSON.getValue()); - dataObject = baseResult.getData(); - Map<String, Object> resultMap = (Map<String, Object>) dataObject; - Map<String, Object> statisticMap = (Map<String, Object>) resultMap.get("statistics"); - Map<String, Object> statistics = Maps.newLinkedHashMap(); - statistics.put("elapsed", Math.round(Double.parseDouble(String.valueOf(statisticMap.get("elapsed"))) * 1000)); - statistics.put("rows_read", statisticMap.get("rows_read")); - baseResult.setData(resultMap.get("data")); - baseResult.setStatistics(statistics); - } else { - baseResult = BaseResultGenerator.generate(Integer.parseInt(results.get("status")), QGWErrorCode.SQL_BUILDER_EXCEPTION.getCode(), null, - results.get("result"), null, null, QueryFormatEnum.JSON.getValue()); - dataObject = baseResult.getData(); - baseResult.setStatus((baseResult.getStatus() < 500) || (baseResult.getStatus() > 400) ? ResultStatusEnum.SERVER_ERROR.getCode() : baseResult.getStatus()); - baseResult.setMessage("SQL Execution Error: ".concat(JsonPath.read(dataObject.toString(), "$.errorMessage"))); - } - return baseResult; - } - - private Map<String, String> clickHouseExecuteHttpPost(String sql) { - String ckQuery = Encodes.urlEncode(sql); - String queryURL = URLUtil.normalize(clickHouseHttpSource.getUrl() + "/?"); - StringBuilder queryParamBuilder = new StringBuilder("user=") - .append(clickHouseHttpSource.getRealTimeAccountUserName()).append("&") - .append("password=").append(clickHouseHttpSource.getRealTimeAccountPin()).append("&") - .append("database=").append(clickHouseHttpSource.getDbName()); - ClickHouseHttpQuery clickHouseHttpQuery = new ClickHouseHttpQuery(); - clickHouseHttpQuery.setSocketTimeOut(httpConfig.getCkRealTimeAccountSocketTimeOut()); - clickHouseHttpQuery.setQueryParameter(queryParamBuilder.toString()); - log.info("DB Engine is :{}, Execute Http Query is: {}", DBTypeEnum.CLICKHOUSE.getValue(), JSON.toJSONString(clickHouseHttpQuery)); - List<NameValuePair> values = URLEncodedUtils.parse(clickHouseHttpQuery.getQueryParameter(), Charset.forName("UTF-8")); - return httpClientService.httpPost(queryURL + URLEncodedUtils.format(values, "utf-8"), sql.concat(" FORMAT ").concat(QueryFormatEnum.JSON.getValue().toUpperCase()), clickHouseHttpQuery.getSocketTimeOut()); - } - - - - /** - * @param sql - * @return com.mesalab.common.base.BaseResult - * @Description hbase SQL测试集在hbase数据源查询并获取查询结果 - * @author wanghao - * @date 2021/11/11 10:01 - */ - private BaseResult generateHbaseBaseResult(String sql) { - BaseResult baseResult = null; - HbaseDialect hbaseDialect = new HbaseDialect(queryBuild(sql)); - baseResult = hbaseDialect.generateBaseResult(converter(sql), Optional.of("OK")); - return baseResult; - } - - private String converter(String sql) { - try { - Select select = (Select) CCJSqlParserUtil.parse(sql); - select.getSelectBody().accept(SQLVisitorUtil.getVisitorOfEscapeMetadataWithDoubleQuote()); - return select.toString(); - } catch (JSQLParserException | RuntimeException e) { - log.error("SQL Syntax Error: Converter SQL Syntax Error, SQL is:{}, Error is:{}", sql, e); - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getCode(), - String.format(QGWErrorCode.SQL_SYNTAX_PARSE_EXCEPTION.getMessage(),e.getMessage())); - } - } - - private QueryProfile queryBuild(String sql) { - return QueryProfile.builder().format(QueryFormatEnum.JSON.getValue()).query(sql).build(); - } - - - private String processFieldVariable(String str) - { - Matcher matcher = pFieldVariable.matcher(str); - StringBuffer sb = new StringBuffer(); - while (matcher.find()) { - matcher.appendReplacement(sb, matcher.group(2)); - } - matcher.appendTail(sb); - return sb.toString(); - } - - private String processOptionalClause(String str, String leftFlag, String rightFlag) - { - String left = ""; - String right = ""; - String center = ""; - boolean leftMark = false; - for (int i = 0; i < str.length(); i++) { - String element = str.substring(i, i + 1); - if (leftFlag.startsWith(element) && leftFlag.equals(str.substring(i, i + leftFlag.length()))) { - left = str.substring(0, i); - leftMark = true; - continue; - } - if (leftMark) { - if (rightFlag.startsWith(element) && rightFlag.equals(str.substring(i, i + rightFlag.length()))) { - right = str.substring(i + rightFlag.length()); - center = str.substring(left.length(), str.length() - (right.length())); - break; - } - } - } - if (StrUtil.isNotEmpty(center)) { - String substring = center.substring(leftFlag.length(), center.length() - leftFlag.length()); - str = left.concat(substring).concat(right); - } - return str; - } - @Data - class DruidQueryParam { - private String query; - private Map<String, String> context = Maps.newHashMap(); - private String resultFormat; - } - -} diff --git a/src/main/java/com/mesalab/services/service/impl/SQLDatasetServiceImpl.java b/src/main/java/com/mesalab/services/service/impl/SQLDatasetServiceImpl.java new file mode 100644 index 00000000..82580f21 --- /dev/null +++ b/src/main/java/com/mesalab/services/service/impl/SQLDatasetServiceImpl.java @@ -0,0 +1,88 @@ +package com.mesalab.services.service.impl; + +import cn.hutool.crypto.digest.DigestUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.google.common.base.CaseFormat; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.common.enums.*; +import com.mesalab.common.utils.sqlparser.SQLHelper; +import com.mesalab.qgw.constant.QGWMessageConst; +import com.mesalab.qgw.dialect.Dialect; +import com.mesalab.qgw.exception.QGWBusinessException; +import com.mesalab.common.exception.CommonErrorCode; +import com.mesalab.qgw.model.basic.*; +import com.mesalab.qgw.service.DatabaseService; +import com.mesalab.services.common.property.SqlPropertySourceFactory; +import com.mesalab.services.service.SQLDatasetService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.PropertySource; +import org.springframework.stereotype.Service; + +import java.lang.reflect.Constructor; + +/** + * TODO + * + * @Classname SQLDatasetServiceImp + * @Date 2023/8/10 13:54 + * @Author wWei + */ +@Service("SQLDatasetService") +@PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) +public class SQLDatasetServiceImpl + implements SQLDatasetService { + private static final Log log = LogFactory.get(); + + public final static String QUERY_ID_SEPARATOR = ":"; + + @Autowired + private DatabaseService databaseService; + + + @Override + public BaseResult getProcessesByQueryId(String queryId) { + Dialect dialect = buildAndGetDialect(queryId); + return dialect.getProcesses(queryId); + } + + @Override + public BaseResult deleteQueryTask(String queryId) { + Dialect dialect = buildAndGetDialect(queryId); + return dialect.executeKillQuery(queryId); + } + + public Dialect buildAndGetDialect(String queryId) { + SQLQueryContext queryContext = new SQLQueryContext(); + queryContext.setDbEngine(parseDBEngineType(queryId)); + return getDialect(queryContext); + } + + private String parseDBEngineType(String queryId) { + for (DBEngineType dbTypeEnum : DBEngineType.values()) { + if (queryId.startsWith(DigestUtil.md5Hex(dbTypeEnum.getValue()).concat(QUERY_ID_SEPARATOR))) { + return dbTypeEnum.getValue(); + } + } + if (queryId.contains(QUERY_ID_SEPARATOR)) { + throw new QGWBusinessException(HttpStatusCodeEnum.NOT_FOUND.getCode(), CommonErrorCode.PARAMETER_ERROR.getCode(), + String.format(CommonErrorCode.PARAMETER_ERROR.getMessage(), QGWMessageConst.QUERY_ID_IS_NOT_SUPPORTED)); + } + return DBEngineType.CLICKHOUSE.getValue(); + } + + private Dialect getDialect(SQLQueryContext queryContext) { + Dialect dialect; + try { + Class dialectClazz = Class.forName("com.mesalab.qgw.dialect." + + CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, queryContext.getDbEngine()) + "Dialect"); + Constructor constructor = dialectClazz.getConstructor(SQLQueryContext.class); + dialect = (Dialect) constructor.newInstance(queryContext); + } catch (ReflectiveOperationException | RuntimeException e) { + log.error("Dialect conversion instance exception:{}", e); + throw new QGWBusinessException(HttpStatusCodeEnum.BAD_REQUEST.getCode(), CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getCode(), + String.format(CommonErrorCode.BAD_REQUEST_SQL_SYNTAX_PARSE_EXCEPTION.getMessage(), e.getMessage())); + } + return dialect; + } +} diff --git a/src/main/java/com/mesalab/services/service/impl/TaskExecuteService.java b/src/main/java/com/mesalab/services/service/impl/TaskExecuteService.java index 78a16931..733f7fc9 100644 --- a/src/main/java/com/mesalab/services/service/impl/TaskExecuteService.java +++ b/src/main/java/com/mesalab/services/service/impl/TaskExecuteService.java @@ -7,35 +7,30 @@ import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.alibaba.fastjson.JSONArray; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.jayway.jsonpath.JsonPath; import com.mesalab.common.entity.BaseResult; import com.mesalab.common.entity.DataTypeMapping; -import com.mesalab.common.enums.QueryOptionEnum; +import com.mesalab.common.enums.QueryOption; import com.mesalab.common.exception.BusinessException; -import com.mesalab.qgw.model.basic.QueryProfile; +import com.mesalab.qgw.model.basic.SQLQueryContext; import com.mesalab.qgw.model.basic.EngineConfigSource; -import com.mesalab.qgw.service.QueryService; -import com.mesalab.qgw.service.MetadataService; +import com.mesalab.qgw.service.SQLSyncQueryService; +import com.mesalab.qgw.service.DatabaseService; import com.mesalab.services.common.property.SqlPropertySourceFactory; import com.mesalab.services.configuration.JobConfig; import com.geedgenetworks.utils.StringUtil; -import org.apache.avro.LogicalType; -import org.apache.avro.Schema; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.EnvironmentAware; import org.springframework.context.annotation.PropertySource; import org.springframework.core.env.Environment; -import org.springframework.scheduling.annotation.Async; -import org.springframework.scheduling.annotation.AsyncResult; import org.springframework.stereotype.Service; import java.util.*; -import java.util.concurrent.Future; -import static com.mesalab.services.service.impl.JobServiceImp.executeUpdate; +import static com.mesalab.services.service.impl.JobExecuteService.markJobFailure; +import static com.mesalab.services.service.impl.JobServiceImpl.executeUpdate; /** * TODO @@ -49,78 +44,51 @@ import static com.mesalab.services.service.impl.JobServiceImp.executeUpdate; public class TaskExecuteService implements EnvironmentAware { private static final Log log = LogFactory.get(); @Autowired - Environment env; + private Environment env; @Autowired - QueryService queryService; + SQLSyncQueryService sqlSyncQueryService; @Autowired - MetadataService metadataService; + DatabaseService databaseService; @Autowired EngineConfigSource engineConfigSource; - @Async("taskExecutor") - public Future<Boolean> executeFieldDiscovery(String id, String logType, String filter, String field, Object metric, Object fn, long currentTotalMetric) { + public List<Map<String, Object>> executeFieldDiscovery(String id, String dataSource, String field, Object metric, Object fn, String filter) { + String fnDefault = "count"; + String column = "*"; + String value = "count"; try { - String fnDefault = "count"; - String column = "*"; - String value = "count"; if (StringUtil.isNotEmpty(metric)) { - String measurements = metadataService.getValueByKeyInSchemaDoc(logType, "measurements"); - List<Map<String, String>> read = JsonPath.read(measurements, "$.aggregates['" + metric + "'][?(@.fn == '" + fn + "')]"); + String measurements = databaseService.getValueByKeyInSchemaDoc(dataSource, "measurements"); + List<Map<String, String>> read = JsonPath.read(measurements, "$.field_discovery_metric['" + metric + "'][?(@.fn == '" + fn + "')]"); if (!read.isEmpty()) { fnDefault = fn.toString(); column = read.get(0).get("column"); value = read.get(0).get("value"); } } - - BaseResult currentTop = queryService.executeQuery(QueryProfile.builder() - .option(QueryOptionEnum.REAL_TIME.getValue()) - .query(buildTopSQL(logType, filter, field, fnDefault, column, value)) + BaseResult currentTop = sqlSyncQueryService.executeQuery(SQLQueryContext.builder() + .option(QueryOption.REAL_TIME.getValue()) + .originalSQL(buildTopSQL(dataSource, filter, field, fnDefault, column, value)) .build()); if (!currentTop.isSuccess()) { - setJobFailed(id); - log.error("Field-Discovery task query currentTopK Error: ", currentTop.getMessage()); - return new AsyncResult<>(false); + markJobFailure(id, "field_discovery task of Job error. message: " + currentTop.getMessage()); + log.error("field_discovery task of Job id is: {}, query currentTopK Error: {}", id, currentTop.getMessage()); + return Lists.newArrayList(); } List<Map<String, Object>> data = (List<Map<String, Object>>) currentTop.getData(); - - BaseResult lastTop = queryService.executeQuery(QueryProfile.builder() - .option(QueryOptionEnum.REAL_TIME.getValue()) - .query(String.format(Objects.requireNonNull(env.getProperty("FIELD_RESULT")), JobConfig.FIELD_DISCOVERY, field, id)) - .build()); - if (!lastTop.isSuccess()) { - setJobFailed(id); - log.error("Field-Discovery task query lastTopK Error: ", lastTop.getMessage()); - return new AsyncResult<>(false); - } - Object lastTotalMetric = 0; - if (StringUtil.isNotEmpty(lastTop.getData())) { - List<Map> map = (List<Map>) lastTop.getData(); - if (StringUtil.isNotEmpty(map.get(0).get("value"))) { - Map item = JSONUtil.toBean(String.valueOf(map.get(0).get("value")), Map.class); - if (StringUtil.isNotEmpty(item)) { - List<Map<String, Object>> topk = (List<Map<String, Object>>) item.get("topk"); - lastTotalMetric = item.get(value); - Map<String, String> metrics = Maps.newHashMap(); - metrics.put(value, "count".equalsIgnoreCase(fnDefault) ? "sum" : fnDefault); - data = mergeData((List<Map<String, Object>>) JSONArray.parse(JSONUtil.toJsonStr(data)), topk, Lists.newArrayList("value"), metrics); - } - } - } - data = sortDataAndSetMaxSize(data, value, DataTypeMapping.LONG, false); - updateResult(id, field, currentTotalMetric, lastTotalMetric, value, data); + return Lists.newArrayList(data); } catch (RuntimeException ex) { - log.error("Field-Discovery task execute error, message is: {}", ex.getMessage()); - setJobFailed(id); + log.error("field_discovery task of Job id is: {}, execute error, message is: {}", id, ex.getMessage()); + markJobFailure(id, ex.getMessage()); } - return new AsyncResult<>(true); + return Lists.newArrayList(); } - @Async("taskExecutor") - public Future<Boolean> executeLongTerm(String id, String sql) { + @Deprecated + public Boolean executeLongTerm(String id, String sql) { try { - BaseResult lastResult = queryService.executeQuery(QueryProfile.builder().query(sql).build()); - BaseResult preResult = queryService.executeQuery(QueryProfile.builder().query(String.format(Objects.requireNonNull(env.getProperty("JOB_RESULT")), id)).build()); + BaseResult lastResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(sql).build()); + BaseResult preResult = sqlSyncQueryService.executeQuery(SQLQueryContext.builder().originalSQL(String.format(Objects.requireNonNull(env.getProperty("JOB_RESULT")), id)).build()); List<Map<String, Object>> lastData = (List<Map<String, Object>>) lastResult.getData(); List<Map<String, Object>> lastMeta = (List<Map<String, Object>>) lastResult.getMeta(); Map preDataFirst = ((List<Map>) preResult.getData()).get(0); @@ -138,21 +106,23 @@ public class TaskExecuteService implements EnvironmentAware { executeUpdate(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_FIELD_VALUE_SINGLE_QUOTATION")), JobConfig.LONG_TERM_COLUMN_FAMILY, JobConfig.LONG_TERM_RESULT, id, JSONUtil.quote(JSONUtil.toJsonStr(result), false).replace("'", "\\'"))); } catch (RuntimeException ex) { log.error("Statistics task execute error, message is: {}", ex.getMessage()); - setJobFailed(id); + setJobFailedLongTerm(id); } - return new AsyncResult<>(true); + return true; } - private void setJobFailed(String id) { + @Deprecated + private void setJobFailedLongTerm(String id) { executeUpdate(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_FIELD_VALUE")), JobConfig.DETAIL, JobConfig.IS_FAILED, id, true)); } + private void topOrRareData(List<Map<String, Object>> lastData, List<Map<String, Object>> lastMeta, Map preDataFirst, Map<String, Object> valueMap) { List<Map<String, Object>> preData; List<String> dimensions = JsonPath.read(JSONUtil.toJsonStr(preDataFirst.get(JobConfig.JOB_PROPERTY)), "$.sub_option.dimension"); Map<String, String> metrics = JsonPath.read(JSONUtil.toJsonStr(preDataFirst.get(JobConfig.JOB_PROPERTY)), "$.sub_option.metric"); Map<String, Boolean> isAsc = JsonPath.read(JSONUtil.toJsonStr(preDataFirst.get(JobConfig.JOB_PROPERTY)), "$.sub_option.isAsc"); - String sortKey = isAsc.keySet().stream().findFirst().get(); + String sortKey = isAsc.keySet().stream().findFirst().orElse(null); if (StringUtil.isNotEmpty(preDataFirst.get(JobConfig.LONG_TERM_RESULT))) { Map<String, Object> item = JSONUtil.toBean(preDataFirst.get(JobConfig.LONG_TERM_RESULT).toString(), Map.class); preData = (List<Map<String, Object>>) item.get("data"); @@ -160,7 +130,7 @@ public class TaskExecuteService implements EnvironmentAware { lastData = mergeData(lastData, preData, dimensions, metrics); Map<String, String> meta = Maps.newHashMap(); lastMeta.forEach(o -> meta.put(String.valueOf(o.get("name")), String.valueOf(o.get("type")))); - lastData = sortDataAndSetMaxSize(lastData, sortKey, meta.get(sortKey), isAsc.values().stream().findFirst().get()); + lastData = sortDataAndSetMaxSize(lastData, sortKey, meta.get(sortKey), isAsc.values().stream().findFirst().orElse(true)); } valueMap.put("data", lastData); valueMap.put("meta", lastMeta); @@ -175,7 +145,7 @@ public class TaskExecuteService implements EnvironmentAware { Map<String, Boolean> isAsc = JsonPath.read(JSONUtil.toJsonStr(preDataFirst.get(JobConfig.JOB_PROPERTY)), "$.sub_option.isAsc"); Map<String, String> meta = Maps.newHashMap(); lastMeta.forEach(o -> meta.put(String.valueOf(o.get("name")), String.valueOf(o.get("type")))); - lastData = appendData(lastData, preData, isAsc.values().stream().findFirst().get()); + lastData = appendData(lastData, preData, isAsc.values().stream().findFirst().orElse(true)); } valueMap.put("meta", lastMeta); valueMap.put("data", lastData); @@ -267,34 +237,32 @@ public class TaskExecuteService implements EnvironmentAware { } private String buildTopSQL(String tableName, String filter, String fieldName, String fn, String column, String value) { - Schema.Field field = metadataService.getSchemaByName(tableName).getField(fieldName); - if (StringUtil.isNotEmpty(field)) { - Schema schema = field.schema(); - String type = schema.getType().getName(); - String logicalType = schema.getProp(LogicalType.LOGICAL_TYPE_PROP); - if (StrUtil.equalsIgnoreCase(Schema.Type.ARRAY.getName(), type) - || StrUtil.equalsIgnoreCase(Schema.Type.ARRAY.getName(), logicalType)) { - return String.format(Objects.requireNonNull(env.getProperty("JOB_TOPK_ARRAY")), - "count".equalsIgnoreCase(fn) ? "SUM" : fn, value, fieldName, fn + "(" + column + ")", tableName, fieldName, StrUtil.isBlankIfStr(filter) ? "" : " AND ".concat(filter), value); - } else if (StrUtil.equalsIgnoreCase("bit", type) - || StrUtil.equalsIgnoreCase("bit", logicalType)) { - - return String.format(Objects.requireNonNull(env.getProperty("JOB_TOPK_BIT")), - "count".equalsIgnoreCase(fn) ? "SUM" : fn, value, fieldName, fn + "(" + column + ")", tableName, StrUtil.isBlankIfStr(filter) ? "" : " WHERE ".concat(filter), value); + List<Map> fields = (List<Map>) databaseService.getSchemaByName(tableName).get("fields"); + for (Map next : fields) { + if (next.get("name").equals(fieldName)) { + String type; + String logicalType; + if (next.get("type") instanceof String) { + type = String.valueOf(next.get("type")); + logicalType = StrUtil.EMPTY; + } else { + Map typeMap = (Map) next.get("type"); + type = String.valueOf(typeMap.get("type")); + logicalType = StrUtil.isNotBlank(String.valueOf(typeMap.get("logicalType"))) ? String.valueOf(typeMap.get("logicalType")) : StrUtil.EMPTY; + } + if (StrUtil.equalsIgnoreCase("array", type) + || StrUtil.equalsIgnoreCase("array", logicalType)) { + return String.format(Objects.requireNonNull(env.getProperty("JOB_TOPK_ARRAY")), + "count".equalsIgnoreCase(fn) ? "SUM" : fn, JobConfig.FIELD_DISCOVERY_TOPK_METRIC_PREFIX, value, fieldName,fieldName,fieldName, fn + "(" + column + ")", tableName, StrUtil.isBlankIfStr(filter) ? "" : " WHERE ".concat(filter), JobConfig.FIELD_DISCOVERY_TOPK_METRIC_PREFIX, value, engineConfigSource.getFieldDiscoveryTopK()); + } else if (StrUtil.equalsIgnoreCase("bit", type) + || StrUtil.equalsIgnoreCase("bit", logicalType)) { + return String.format(Objects.requireNonNull(env.getProperty("JOB_TOPK_BIT")), + "count".equalsIgnoreCase(fn) ? "SUM" : fn, JobConfig.FIELD_DISCOVERY_TOPK_METRIC_PREFIX, value, fieldName, fn + "(" + column + ")", tableName, StrUtil.isBlankIfStr(filter) ? "" : " WHERE ".concat(filter), JobConfig.FIELD_DISCOVERY_TOPK_METRIC_PREFIX, value, engineConfigSource.getFieldDiscoveryTopK()); + } } - } return String.format(Objects.requireNonNull(env.getProperty("JOB_TOPK_DEFAULT")), - fieldName, fn, column, value, tableName, StrUtil.isBlankIfStr(filter) ? "" : " WHERE ".concat(filter), value); - } - - private void updateResult(String id, String field, long currentTotalMetric, Object lastTotalMetric, String value, List<Map<String, Object>> topList) { - Map<String, Object> map = Maps.newHashMap(); - map.put("topk", topList); - map.put("distinct_count", Math.min(topList.size(), 100)); - map.put(value, currentTotalMetric + Long.parseLong(lastTotalMetric.toString())); - String result = JSONUtil.quote(JSONUtil.toJsonStr(map), false).replace("'", "\\'"); - executeUpdate(String.format(Objects.requireNonNull(env.getProperty("JOB_UPDATE_FIELD_VALUE_SINGLE_QUOTATION")), JobConfig.FIELD_DISCOVERY, field, id, result)); + fieldName, fn, column, JobConfig.FIELD_DISCOVERY_TOPK_METRIC_PREFIX, value, tableName, StrUtil.isBlankIfStr(filter) ? "" : " WHERE ".concat(filter), JobConfig.FIELD_DISCOVERY_TOPK_METRIC_PREFIX, value, engineConfigSource.getFieldDiscoveryTopK()); } @Override diff --git a/src/main/java/com/mesalab/services/service/impl/UnstructuredServiceImpl.java b/src/main/java/com/mesalab/services/service/impl/UnstructuredServiceImpl.java deleted file mode 100644 index e3fad8d0..00000000 --- a/src/main/java/com/mesalab/services/service/impl/UnstructuredServiceImpl.java +++ /dev/null @@ -1,222 +0,0 @@ -package com.mesalab.services.service.impl; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.alibaba.fastjson2.JSON; -import com.clearspring.analytics.util.Lists; -import com.google.common.base.Stopwatch; -import com.google.common.collect.Maps; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.entity.BaseResultGenerator; -import com.mesalab.common.enums.QueryFormatEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.qgw.constant.QGWMessageConst; -import com.mesalab.qgw.exception.QGWBusinessException; -import com.mesalab.qgw.model.basic.QueryProfile; -import com.mesalab.qgw.model.basic.ClickHouseHttpSource; -import com.mesalab.qgw.model.basic.EngineConfigSource; -import com.mesalab.qgw.service.QueryService; -import com.mesalab.qgw.service.MetadataService; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.common.enums.UnstructuredDataType; -import com.mesalab.services.common.property.SqlPropertySourceFactory; -import com.mesalab.services.service.UnstructuredService; -import com.geedgenetworks.utils.StringUtil; -import org.apache.avro.Schema; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.EnvironmentAware; -import org.springframework.context.annotation.PropertySource; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; - - -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -@Service("UnstructuredService") -@PropertySource(value = "classpath:http-sql-template.sql", factory = SqlPropertySourceFactory.class) -public class UnstructuredServiceImpl implements UnstructuredService, EnvironmentAware { - - private static final Log log = LogFactory.get(); - private final static Map<String, String> fileType = Maps.newHashMap(); - private static final String COMMONFILE = "common"; - private static final String[] tables = new String[]{"session_record", "security_event", "transaction_record"}; - static { - fileType.put("common_packet_capture_file", UnstructuredDataType.PCAP.getType()); - fileType.put("rtp_pcap_path", UnstructuredDataType.PCAP.getType()); - fileType.put("packet_url", UnstructuredDataType.PCAP.getType()); - fileType.put("mail_eml_file", UnstructuredDataType.MAIL.getType()); - fileType.put("http_request_body", UnstructuredDataType.HTTP.getType()); - fileType.put("http_response_body", UnstructuredDataType.HTTP.getType()); - } - - @Autowired - private MetadataService metadataService; - @Autowired - private ClickHouseHttpSource clickHouseHttpSource; - @Autowired - private EngineConfigSource engineConfigSource; - @Autowired - private QueryService queryService; - private Environment env; - - @Override - public Map<String, Object> getUnstructuredFields() { - Map<String, Object> fileTypeInSchema = Maps.newHashMap(); - List<String> allTable = metadataService.getAllTable(); - for (String tableName : allTable) { - Schema schema = metadataService.getSchemaByName(tableName); - Map schemaMap = JSON.parseObject(schema.toString(), Map.class); - List<String> fieldList = new ArrayList<>(); - if (!clickHouseHttpSource.getDbName().equals(schema.getNamespace())) continue; - List<Map> fields = (List<Map>) schemaMap.get("fields"); - for (Map field : fields) { - Map schemaDoc = JSON.parseObject(field.get("doc").toString(), Map.class); - if (schemaDoc.containsKey("constraints") && schemaDoc.get("visibility").equals("enabled")) { - Map<String, String> SchemaConstraints = (Map<String, String>) schemaDoc.get("constraints"); - String typeStr = SchemaConstraints.containsKey("type") ? SchemaConstraints.get("type") : StringUtil.EMPTY; - if (typeStr.equals("file") || typeStr.equals("files")) { - fieldList.add((String) field.get("name")); - } - } - } - - Map<String, List<String>> fileTypeMap = Maps.newHashMap(); - if (fieldList.size() != 0) { - for (String field : fieldList) { - String fileTypeStr = StringUtil.isNotEmpty(fileType.get(field)) ? fileType.get(field).toString() : COMMONFILE; - if (fileTypeMap.containsKey(fileTypeStr)) { - List<String> preList = fileTypeMap.get(fileTypeStr); - preList.add(field); - fileTypeMap.put(fileTypeStr, preList); - } else { - List<String> newList = new ArrayList<>(); - newList.add(field); - fileTypeMap.put(fileTypeStr, newList); - } - } - } - if (!fileTypeMap.isEmpty()) { - fileTypeInSchema.put(tableName, fileTypeMap); - } - } - return fileTypeInSchema; - } - - @Override - public BaseResult getUnstructuredData(String param, ComDSLObject dslObject) { - Stopwatch watch = Stopwatch.createStarted(); - List<Map> dataList = Lists.newArrayList(); - String dataSource = dslObject.getQuery().getDataSource(); - validationDataSource(dataSource); - Map<String, Object> fileTypeInSchema = getUnstructuredFields(); - if (!fileTypeInSchema.containsKey(dataSource) && !dataSource.equals(clickHouseHttpSource.getDbName())) { - return build(dataList, watch); - } - int limit = StringUtil.isBlank(dslObject.getQuery().getLimit()) ? engineConfigSource.getDefaultResultNum() : Integer.parseInt(dslObject.getQuery().getLimit()); - if (!param.equalsIgnoreCase(UnstructuredDataType.ALL.getType())) { - if (!dataSource.equals(clickHouseHttpSource.getDbName())) { - dataList = getData(fileTypeInSchema, limit, param, dslObject, dataSource); - } else { - for (String tableName : fileTypeInSchema.keySet()) { - dataList.addAll(getData(fileTypeInSchema, limit, param, dslObject, tableName)); - } - } - } else { - if (!dataSource.equals(clickHouseHttpSource.getDbName())) { - Map<String, List<String>> fileType = (Map<String, List<String>>) fileTypeInSchema.get(dataSource); - for (String s : fileType.keySet()) { - dataList.addAll(getData(fileTypeInSchema, limit, s, dslObject, dataSource)); - } - } else { - for (String tableName : fileTypeInSchema.keySet()) { - Map<String, Object> fileType = (Map<String, Object>) fileTypeInSchema.get(tableName); - for (String s : fileType.keySet()) { - dataList.addAll(getData(fileTypeInSchema, limit, s, dslObject, tableName)); - } - } - } - } - - List<Map> distinctList = dataList.stream() - .filter(distinctByValue(x -> x.get("file_path"))) - .collect(Collectors.toList()); - - distinctList = distinctList.size() > limit ? distinctList.subList(0, Integer.parseInt(dslObject.getQuery().getLimit())) : distinctList; - return build(distinctList, watch); - } - - private BaseResult build(List<Map> dataList, Stopwatch watch) { - BaseResult baseResult; - Map statistics = Maps.newHashMap(); - statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); - statistics.put("result_rows", dataList.size()); - statistics.put("result_bytes", dataList.toString().getBytes().length); - baseResult = BaseResultGenerator.success("ok", dataList, statistics); - return baseResult; - } - - private List<Map> getData(Map<String, Object> fileTypeMap, int limit, String param, ComDSLObject dslObject, String dataSource) { - List<Map> dataResult = Lists.newArrayList(); - Map<String, List<String>> tableFileType = (Map<String, List<String>>) fileTypeMap.get(dataSource); - if (tableFileType.containsKey(param)) { - List<String> listFields = tableFileType.get(param); - for (String field : listFields) { - String sql = String.format( - Arrays.asList(tables).contains(dataSource) ? Objects.requireNonNull(env.getProperty("UNSTRUCTURED_DATA_PATH")) : Objects.requireNonNull(env.getProperty("UNSTRUCTURED_DATA_PATH_OLD")) - , field, dataSource - , generateWhereOfTime(Arrays.asList(tables).contains(dataSource) ? "recv_time" : "common_recv_time", dslObject.getQuery()) - , field, limit); - BaseResult result = queryService.executeQuery(queryBuild(sql)); - if (!result.isSuccess()) { - log.error("Query unstructured data status error: {}", result.getMessage()); - throw new QGWBusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOWN_EXCEPTION.getCode(), "Query unstructured data status error: " + result.getMessage()); - } - List<Map> datalist = (List<Map>) result.getData(); - for (Map data : datalist) { - data.put("log_type", dataSource); - dataResult.add(data); - } - } - } - return dataResult; - } - - private String generateWhereOfTime(String timeKey, ComDSLObject.Query query) { - String[] intervals = getIntervals(query.getParameters().getIntervals()); - String whereOfTime = timeKey + " >= UNIX_TIMESTAMP('" + intervals[0] + "') AND " + timeKey + " < UNIX_TIMESTAMP('" + intervals[1] + "')"; - return whereOfTime; - } - - private String[] getIntervals(List<String> intervals) { - return intervals.get(0).split("/"); - } - - private static <T> Predicate<T> distinctByValue(Function<? super T, Object> keyExtractor) { - Map<Object, Boolean> seen = new ConcurrentHashMap<>(); - return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null; - } - - - private QueryProfile queryBuild(String sql) { - return QueryProfile.builder().format(QueryFormatEnum.JSON.getValue()).query(sql).build(); - } - - private void validationDataSource(String dataSource) { - List<String> allTable = metadataService.getAllTable(); - if (!allTable.contains(dataSource) && !dataSource.equals(clickHouseHttpSource.getDbName())) { - throw new QGWBusinessException(ResultStatusEnum.BAD_REQUEST.getCode(), ResultCodeEnum.PARAMETER_ERROR.getCode(), - String.format(ResultCodeEnum.PARAMETER_ERROR.getMessage(), QGWMessageConst.DATASOURCE_NOT_EXIST)); - } - } - - @Override - public void setEnvironment(Environment environment) { - this.env = environment; - } -} diff --git a/src/main/resources/dsl-sql-template.sql b/src/main/resources/dsl-sql-template.sql index 4533a2bc..9906ba88 100644 --- a/src/main/resources/dsl-sql-template.sql +++ b/src/main/resources/dsl-sql-template.sql @@ -4,9 +4,6 @@ SELECT SUM(asymmetric_c2s_flows + asymmetric_s2c_flows) AS asymmetric_flows, SUM #sql("APPLICATION_AND_PROTOCOL_SUMMARY_TRAFFIC_STAT") SELECT SUM(fragmentation_packets) AS fragmentation_packets, SUM(total_bytes) AS total_bytes, SUM(total_packets) AS total_packets, SUM(total_sessions) AS total_sessions, AVG(total_bytes) * 8 / $granularity_seconds AS data_rate FROM ( SELECT SUM(c2s_fragments + s2c_fragments) AS fragmentation_packets, SUM(c2s_bytes + s2c_bytes) AS total_bytes, SUM(c2s_pkts + s2c_pkts) AS total_packets, SUM(sessions) AS total_sessions FROM $table WHERE $intervals_and_filter AND protocol_stack_id = 'ETHERNET' GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), '$granularity')) LIMIT 1 #end -#sql("APPLICATION_AND_PROTOCOL_SUMMARY_UNIQUE_CLIENT_IP_STAT") -SELECT APPROX_COUNT_DISTINCT_DS_HLL(client_ip_sketch) AS uniq_client_ip FROM $table WHERE $intervals_and_filter AND protocol_stack_id = 'ETHERNET' LIMIT 1 -#end #sql("APPLICATION_AND_PROTOCOL_SUMMARY_TCP_STAT") SELECT SUM(c2s_tcp_retransmitted_pkts + s2c_tcp_retransmitted_pkts) AS tcp_retransmissions_packets, SUM(c2s_pkts + s2c_pkts) AS tcp_total_packets FROM $table WHERE $intervals_and_filter AND RIGHT(protocol_stack_id, 4) = '.TCP' LIMIT 1 #end @@ -17,23 +14,38 @@ SELECT SUM(c2s_bytes + s2c_bytes) as unknown_app_bytes FROM $table WHERE $interv SELECT protocol_stack_id, SUM(sessions) as sessions,SUM(c2s_bytes) as c2s_bytes, SUM(c2s_pkts) as c2s_pkts, SUM(s2c_bytes) as s2c_bytes, SUM(s2c_pkts) as s2c_pkts FROM $table WHERE $intervals_and_filter GROUP BY protocol_stack_id #end #sql("APPLICATION_AND_PROTOCOL_TREE_THROUGHPUT") -SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), '$granularity', 'zero')) as stat_time, protocol_stack_id as type, sum(c2s_bytes + s2c_bytes) as bytes from $table where $intervals_and_filter group by FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), '$granularity', 'zero')), protocol_stack_id order by stat_time asc +SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), '$granularity', 'zero')) as stat_time, protocol_stack_id as type, sum(c2s_bytes + s2c_bytes) as bytes, SUM(c2s_bytes + s2c_bytes) * 8 / $granularity_seconds AS bit_rate from $table where $intervals_and_filter group by FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), '$granularity', 'zero')), protocol_stack_id order by stat_time asc #end #sql("APPLICATION_AND_PROTOCOL_TOP_APPS") -SELECT app_name as app_name, SUM(bytes) as bytes, SUM(sessions) as sessions, MAX(bytes_rate) as max_rate, AVG(bytes_rate) as avg_rate FROM (SELECT app_name as app_name, SUM(sessions) as sessions, SUM(c2s_bytes + s2c_bytes) as bytes, SUM(c2s_bytes + s2c_bytes) * 8 / $granularity_seconds as bytes_rate FROM $table WHERE $intervals_and_filter AND notEmpty(app_name) GROUP BY app_name ORDER BY bytes DESC ) GROUP BY app_name ORDER BY bytes DESC $limit +SELECT app_name as app_name, SUM(bytes) as bytes, SUM(sessions) as sessions, MAX(bit_rate) as max_rate, SUM(bytes) * 8/$interval_seconds as avg_rate FROM (SELECT app_name as app_name, SUM(sessions) as sessions, SUM(c2s_bytes + s2c_bytes) as bytes, SUM(c2s_bytes + s2c_bytes) * 8 / $granularity_seconds as bit_rate FROM $table WHERE $intervals_and_filter AND notEmpty(app_name) GROUP BY app_name, TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), '$granularity') ORDER BY bytes DESC ) GROUP BY app_name ORDER BY bytes DESC $limit #end #sql("APPLICATION_AND_PROTOCOL_APP_RELATED_INTERNAL_IPS") SELECT if(bitAnd(flags, 8) = 8, client_ip, server_ip) AS ip, SUM(sent_bytes + received_bytes) AS bytes FROM $table WHERE $intervals_and_filter GROUP BY ip ORDER BY bytes DESC $limit #end #sql("APPLICATION_AND_PROTOCOL_APP_THROUGHPUT") -SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'$granularity','zero')) AS stat_time, app_name, SUM(c2s_bytes + s2c_bytes) AS bytes FROM $table WHERE $intervals_and_filter GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'$granularity','zero')), app_name ORDER BY stat_time ASC +SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'$granularity','zero')) AS stat_time, app_name, SUM(c2s_bytes + s2c_bytes) AS bytes, SUM(c2s_bytes + s2c_bytes) * 8 / $granularity_seconds AS bit_rate FROM $table WHERE $intervals_and_filter GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'$granularity','zero')), app_name ORDER BY stat_time ASC #end #sql("APPLICATION_AND_PROTOCOL_APP_SUMMARY") SELECT app_name, SUM(sessions) AS sessions, SUM(c2s_bytes + s2c_bytes) AS bytes, SUM(s2c_bytes) AS received_bytes, SUM(c2s_bytes) AS sent_bytes, SUM(c2s_pkts + s2c_pkts) AS packets, SUM(c2s_pkts) AS sent_packets, SUM(s2c_pkts) AS received_packets FROM $table WHERE $intervals_and_filter GROUP BY app_name #end -#sql("REAL_TIME_DATA_ANALYTICS_SUBSCRIBER_ID_RELATE_IP") -SELECT account AS SUBSCRIBER_ID, framed_ip AS IP, first_found_time AS FIRST_FOUND_TIME, last_update_time AS LAST_FOUND_TIME, vsys_id as VSYS_ID FROM $table WHERE $intervals_and_filter AND acct_status_type != 2 $limit +#sql("DATAPATH_TELEMETRY_RECORD") +SELECT * FROM $table WHERE $intervals_and_filter ORDER BY timestamp_us ASC +#end +#sql("TRAFFIC_SPECTRUM_SUMMARY") +SELECT direction, SUM(bytes) AS total_bytes, SUM(sessions) AS total_sessions, SUM(pkts) AS total_packets, SUM(IF(app = 'unknown', bytes, 0)) AS unknown_app_bytes, SUM(asymmetric_c2s_flows + asymmetric_s2c_flows) AS asymmetric_flows, SUM(c2s_fragments + s2c_fragments) AS fragmentation_packets FROM $table WHERE $intervals_and_filter GROUP BY direction +#end +#sql("TRAFFIC_SPECTRUM_UNIQ_IP") +SELECT COUNT(DISTINCT(IF(direction = 'Outbound', client_ip, NULL))) AS internal_uniq_client_ip, COUNT(DISTINCT(IF(direction = 'Outbound', server_ip, NULL))) AS external_uniq_server_ip, COUNT(DISTINCT(IF(direction = 'Inbound', server_ip, NULL))) AS internal_uniq_server_ip, COUNT(DISTINCT(IF(direction = 'Inbound', client_ip, NULL))) AS external_uniq_client_ip FROM $table WHERE $intervals_and_filter +#end +#sql("TRAFFIC_SPECTRUM_APP_DISTRIBUTION_SERVER_IP") +SELECT server_ip as value, SUM(bytes) as bytes, SUM(sessions) as sessions, SUM(pkts) as packets FROM $table WHERE $intervals_and_filter GROUP BY server_ip ORDER BY bytes DESC $limit +#end +#sql("TRAFFIC_SPECTRUM_APP_DISTRIBUTION_SERVER_DOMAIN") +SELECT server_domain as value, SUM(bytes) as bytes, SUM(sessions) as sessions, SUM(pkts) as packets FROM $table WHERE $intervals_and_filter AND notEmpty(server_domain) GROUP BY server_domain ORDER BY bytes DESC $limit +#end +#sql("TRAFFIC_SPECTRUM_CLIENT_IP_CONNECT_APPLICATION_USAGE") +SELECT direction, SUM(temp_sessions) AS sessions, SUM(temp_bytes) AS bytes, SUM(temp_packets) AS packets, MAX(temp_rate) AS max_rate, ROUND(SUM(temp_bytes) * 8 /$interval_seconds) AS avg_rate, client_ip AS client_ip, client_country AS client_country, app AS app, app_category AS app_category FROM( SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(recv_time), INTERVAL $granularity_seconds SECOND))) AS stat_time, IF(client_ip IN ( SELECT client_ip FROM $table WHERE $intervals_and_filter AND notEmpty(app) GROUP BY client_ip,app ORDER BY SUM(bytes) DESC $limit), client_ip, 'Other') AS client_ip, IF(client_country IN ( SELECT anyLast(client_country) FROM $table WHERE $intervals_and_filter AND notEmpty(app) GROUP BY client_ip,app ORDER BY SUM(bytes) DESC $limit), client_country, 'Other') AS client_country, IF(app IN ( SELECT app FROM $table WHERE $intervals_and_filter AND notEmpty(app) GROUP BY client_ip,app ORDER BY SUM(bytes) DESC $limit), app, 'Other') AS app, IF(app_category IN ( SELECT anyLast(app_category) FROM $table WHERE $intervals_and_filter AND notEmpty(app) GROUP BY client_ip,app ORDER BY SUM(bytes) DESC $limit), app_category, 'Other') AS app_category, direction, SUM(sessions) AS temp_sessions, SUM(bytes) AS temp_bytes, SUM(pkts) AS temp_packets, ROUND(SUM(bytes) * 8 / $granularity_seconds) AS temp_rate FROM $table WHERE $intervals_and_filter GROUP BY stat_time, client_ip, client_country, app, app_category, direction) GROUP BY client_ip, client_country, app, app_category, direction ORDER BY bytes DESC #end -#sql("REAL_TIME_DATA_ANALYTICS_MOBILE_IDENTITY_RELATE_TEID") -SELECT uplink_teid AS teid, apn, phone_number, imsi, imei, last_update_time,vsys_id FROM $table WHERE $intervals_and_filter AND msg_type = 1 and uplink_teid !=0 order by last_update_time desc $limit +#sql("TRAFFIC_SPECTRUM_NETWORK_THROUGHPUT_TREND") +SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(recv_time, '$granularity', 'zero')) AS stat_time, ROUND(RATE(bytes,$granularity_seconds,1)) * 8 AS avg_bits_per_sec, ROUND(RATE(bytes,$granularity_seconds,1)) AS avg_bytes_per_sec, ROUND(RATE(pkts,$granularity_seconds,1)) AS avg_pkts_per_sec, ROUND(RATE(sessions,$granularity_seconds,1)) AS avg_sessions_per_sec FROM $table WHERE $intervals_and_filter GROUP BY stat_time ORDER BY stat_time ASC $limit #end
\ No newline at end of file diff --git a/src/main/resources/dsl-validation.json b/src/main/resources/dsl-validation.json index f54eb056..715ac00f 100644 --- a/src/main/resources/dsl-validation.json +++ b/src/main/resources/dsl-validation.json @@ -1,12 +1,11 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "title": "服务推荐", "description": "查询", "type": "object", "properties": { "clientId": { "description": "唯一标识符", - "maximum": 2147483647, - "minimum": -2147483648, "type": [ "null", "integer" @@ -21,12 +20,10 @@ "properties": { "queryType": { "description": "查询类型", - "maxLength": 65535, "type": "string" }, "dataSource": { "description": "查询数据源", - "maxLength": 65535, "type": "string" }, "parameters": { @@ -47,19 +44,16 @@ "properties": { "type": { "description": "匹配符号", - "maxLength": 65535, "type": "string" }, "fieldKey": { "description": "查询字段", - "maxLength": 65535, "type": "string" }, "fieldValues": { "description": "查询参数", "type": "array", "items": { - "maxLength": 65535, "type": "string" } } @@ -76,19 +70,16 @@ "properties": { "type": { "description": "匹配符号", - "maxLength": 65535, "type": "string" }, "fieldKey": { "description": "查询字段", - "maxLength": 65535, "type": "string" }, "fieldValues": { "description": "查询参数", "type": "array", "items": { - "maxLength": 65535, "type": "string" } } @@ -108,21 +99,16 @@ "properties": { "type": { "description": "匹配符号", - "maxLength": 65535, "type": "string" }, "fieldKey": { "description": "查询字段", - "maxLength": 65535, "type": "string" }, "fieldValues": { "description": "查询参数", "type": "array", "items": { - "maximum": 2147483647, - "minimum": -2147483648, - "maxLength": 65535, "type": [ "integer", "string" @@ -141,21 +127,16 @@ "properties": { "type": { "description": "匹配符号", - "maxLength": 65535, "type": "string" }, "fieldKey": { "description": "查询字段", - "maxLength": 65535, "type": "string" }, "fieldValues": { "description": "查询参数", "type": "array", "items": { - "maximum": 2147483647, - "minimum": -2147483648, - "maxLength": 65535, "type": [ "integer", "string" @@ -169,11 +150,9 @@ "description": "查询时间", "type": "array", "items": { - "maxLength": 65535, "type": "string" }, "additionalItems": { - "maxLength": 65535, "type": "string" } }, @@ -186,12 +165,10 @@ "properties": { "type": { "description": "匹配符号", - "maxLength": 65535, "type": "string" }, "fieldKey": { "description": "排序字段", - "maxLength": 65535, "type": "string" } } @@ -202,12 +179,10 @@ "properties": { "type": { "description": "匹配符号", - "maxLength": 65535, "type": "string" }, "fieldKey": { "description": "排序字段", - "maxLength": 65535, "type": "string" } } @@ -215,9 +190,6 @@ }, "limit": { "description": "查询条数", - "maxLength": 65535, - "maximum": 2147483647, - "minimum": 0, "type": [ "integer", "string" diff --git a/src/main/resources/http-sql-template.sql b/src/main/resources/http-sql-template.sql index c81bfb98..4be71868 100644 --- a/src/main/resources/http-sql-template.sql +++ b/src/main/resources/http-sql-template.sql @@ -1,109 +1,9 @@ -#sql("NETWORK_OVERVIEW_ASYMMETRIC_FLOWS_STAT") -SELECT SUM(asymmetric_c2s_flows + asymmetric_s2c_flows) AS asymmetric_flows, SUM(closed_sessions) AS total_session_used_on_asymmetric_flows FROM traffic_general_stat WHERE %s %s LIMIT 1 -#end - -#sql("NETWORK_OVERVIEW_STAT") -SELECT APPROX_COUNT_DISTINCT_DS_HLL(client_ip_sketch) AS uniq_client_ip, SUM(c2s_fragments + s2c_fragments) AS fragmentation_packets, SUM(c2s_bytes + s2c_bytes) AS total_bytes, SUM(c2s_pkts + s2c_pkts) AS total_packets, SUM(sessions) AS total_sessions, (SUM(c2s_bytes + s2c_bytes) * 8)/(%s -%s) AS data_rate FROM %s WHERE %s %s AND protocol_stack_id = '%s' LIMIT 1 -#end - -#sql("NETWORK_OVERVIEW_TCP_STAT") -SELECT SUM(c2s_tcp_retransmitted_pkts + s2c_tcp_retransmitted_pkts) AS tcp_retransmissions_packets, SUM(c2s_pkts + s2c_pkts) AS tcp_total_packets FROM %s WHERE %s %s AND RIGHT(protocol_stack_id, 4) = '.TCP' LIMIT 1 -#end - -#sql("NETWORK_OVERVIEW_APP_STAT") -SELECT SUM(c2s_bytes + s2c_bytes) as unknown_app_bytes FROM %s WHERE %s %s AND app_name = 'unknown' LIMIT 1 -#end - -#sql("PROTOCOL_TREE_SUMMARY") -SELECT protocol_stack_id, SUM(sessions) as sessions,SUM(c2s_bytes) as c2s_bytes, SUM(c2s_pkts) as c2s_pkts, SUM(s2c_bytes) as s2c_bytes, SUM(s2c_pkts) as s2c_pkts FROM %s WHERE %s %s GROUP BY protocol_stack_id -#end - -#sql("PROTOCOL_DATA_RATE_SUMMARY") -(SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_stack_id as type, sum(c2s_bytes + s2c_bytes) as bytes from %s where %s %s and protocol_stack_id = '%s' group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_stack_id order by stat_time asc) -union all -(SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_stack_id as type, sum(c2s_bytes + s2c_bytes) as bytes from %s where %s %s and protocol_stack_id like CONCAT('%s','.%s') and LENGTH(protocol_stack_id) = LENGTH(REPLACE(protocol_stack_id,'.','')) + 1 + %s group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_stack_id order by stat_time asc) -#end - -#sql("APP_DATA_SUMMARY") -SELECT app_name as app_name, SUM(bytes) as bytes, SUM(sessions) as sessions, MAX(bytes_rate) as max_rate, AVG(bytes_rate) as avg_rate FROM (SELECT app_name as app_name, SUM(sessions) as sessions, SUM(c2s_bytes + s2c_bytes) as bytes, SUM(c2s_bytes + s2c_bytes) * 8 / %s as bytes_rate FROM %s WHERE %s %s AND notEmpty(app_name) GROUP BY app_name ORDER BY bytes DESC ) GROUP BY app_name ORDER BY bytes DESC %s -#end - -#sql("APP_INTERNAL_IP_SUMMARY") -SELECT if(bitAnd(flags, 8) = 8, client_ip, server_ip) AS ip, SUM(sent_bytes + received_bytes) AS bytes FROM %s WHERE %s %s GROUP BY ip ORDER BY bytes DESC %s -#end - -#sql("APP_DATA_RATE_SUMMARY") -SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/ 1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, app_name , sum(c2s_bytes + s2c_bytes) as bytes from %s where %s %s group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/ 1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss'), app_name order by stat_time asc -#end - -#sql("APP_TRAFFIC_SUMMARY") -SELECT app_name, SUM(sessions) as sessions, sum(c2s_bytes + s2c_bytes) as bytes, sum(s2c_bytes) as received_bytes, sum(c2s_bytes) as sent_bytes, sum(c2s_pkts + s2c_pkts) as packets, sum(c2s_pkts) as sent_packets, sum(s2c_pkts) as received_packets from %s where %s %s group by app_name -#end - -#sql("ENTITY_ACTIVE_CLIENT_IP") -select client_ip as client_ip, vsys_id as vsys_id from %s where %s %s AND notEmpty(client_ip) GROUP BY client_ip, app, vsys_id ORDER BY COUNT(1) DESC LIMIT %s -#end - -#sql("ENTITY_UDP_SESSION") -select server_ip as server_ip,vsys_id as vsys_id ,COUNT(1) as sessions from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'UDP') and server_port in (53 , 443) group by server_ip,vsys_id order by sessions desc limit %s -#end - -#sql("ENTITY_UDP_UNIQ_CLIENT_IPS") -select server_ip as server_ip, vsys_id as vsys_id from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'UDP') group by server_ip,vsys_id order by COUNT(DISTINCT(client_ip)) desc limit %s -#end - -#sql("ENTITY_TCP_SESSION") -select server_ip as server_ip,vsys_id as vsys_id ,COUNT(1) as sessions from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'TCP') group by server_ip,vsys_id order by sessions desc limit %s -#end - -#sql("ENTITY_TCP_UNIQ_CLIENT_IPS") -select server_ip as server_ip, vsys_id as vsys_id from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'TCP') group by server_ip, vsys_id order by COUNT(DISTINCT(client_ip)) desc limit %s -#end - -#sql("TOP_ENTITY_TCP_UNIQ_CLIENT_IPS") -select COUNT(DISTINCT(client_ip)) as client_ips from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'TCP') AND server_ip in (select server_ip from %s as cc where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'TCP') group by server_ip order by COUNT(DISTINCT(client_ip)) desc limit %s) -#end - -#sql("TOP_ENTITY_UDP_UNIQ_CLIENT_IPS") -select COUNT(DISTINCT(client_ip)) as client_ips from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'UDP') AND server_ip in (select server_ip from %s as cc where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'UDP') group by server_ip order by COUNT(DISTINCT(client_ip)) desc limit %s) -#end - -#sql("TOTAL_ENTITY_UDP_SESSION") -select COUNT(1) as sessions from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'UDP') and server_port in (53 , 443) limit 1 -#end - -#sql("TOTAL_ENTITY_UDP_UNIQ_CLIENT_IPS") -select COUNT(DISTINCT(client_ip)) as uniq_client_ips from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'UDP') limit 1 -#end - -#sql("TOTAL_ENTITY_TCP_SESSION") -select COUNT(1) as sessions from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'TCP') limit 1 -#end - -#sql("TOTAL_ENTITY_TCP_UNIQ_CLIENT_IPS") -select COUNT(DISTINCT(client_ip)) as uniq_client_ips from %s where %s AND (arrayElement(splitByString('.',protocol_path),length(splitByString('.',protocol_path))) = 'TCP') limit 1 -#end - -#sql("ENTITY_TOP_SNI") -select ssl_sni, SUM(sent_bytes + received_bytes) AS bytes, COUNT(1) AS sessions,vsys_id as vsys_id from %s where %s and notEmpty(ssl_sni) and decoded_as = 'SSL' group by ssl_sni,vsys_id order by COUNT(1) desc limit %s -#end - -#sql("ENTITY_SNI_TOTAL") -select SUM(sent_bytes + received_bytes) AS bytes, COUNT(1) AS sessions from %s where %s and decoded_as = 'SSL' limit 1 -#end - #sql("SYSTEM_STORAGE_QUOTA") -SELECT type, SUM(used_size) as used_size, SUM(max_size) * 7 / 10 as max_size, TIME_FORMAT(MILLIS_TO_TIMESTAMP(ANY_VALUE(last_storage) * 1000), 'YYYY-MM-dd') as first_storage FROM - (SELECT log_type as type, LATEST(used_size) as used_size, LATEST(max_size) as max_size, LATEST(last_storage) as last_storage FROM sys_storage_log WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '1' HOUR AND data_center != '' GROUP BY data_center, log_type) -GROUP BY type +SELECT log_type AS type, SUM(used_size) AS used_size, SUM(total_allocated_size) AS total_allocated_size, MIN(since_time) AS since_time FROM (SELECT log_type, used_size, total_allocated_size, DATE(FROM_UNIXTIME(since_time)) AS since_time, ROW_NUMBER() OVER (PARTITION BY log_type ORDER BY generated_time DESC) AS row_num FROM sys_storage_event WHERE generated_time >= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 WEEK)) ) AS subquery WHERE row_num = 1 GROUP BY log_type #end #sql("SYSTEM_DAILY_TREND_OF_STORAGE") -select TIME_FORMAT(__time, 'YYYY-MM-dd') as stat_time, log_type as type, sum(aggregate_size) as used_size from sys_storage_log where __time >= '%s' and __time < '%s' group by TIME_FORMAT(__time, 'YYYY-MM-dd'), log_type -#end - -#sql("RELATION_SUBSCRIBER_ID") -SELECT account AS SUBSCRIBER_ID,framed_ip AS IP,first_found_time AS FIRST_FOUND_TIME,last_update_time AS LAST_FOUND_TIME,vsys_id as VSYS_ID FROM tsg_galaxy.relation_account_framedip WHERE 1=1 %s AND acct_status_type != 2 LIMIT %s +SELECT DATE(FROM_UNIXTIME(generated_time)) AS stat_time, log_type AS type, sum( bytes ) AS used_size FROM sys_storage_event WHERE generated_time >= UNIX_TIMESTAMP('%s') AND generated_time < UNIX_TIMESTAMP('%s') GROUP BY stat_time, type ORDER BY stat_time ASC #end #sql("SCHEMA_STORAGE_SIZE") @@ -114,25 +14,10 @@ SELECT name AS field, IFNULL(SUM(data_compressed_bytes), 0) AS bytes FROM column SELECT name, engine_full AS table_ttl FROM tables WHERE name ='%s_local' AND "database" = '%s' AND "engine" ='MergeTree' #end -#sql("GTPC_KNOWLEDGE_BASE") -SELECT uplink_teid AS teid, apn, phone_number, imsi, imei, last_update_time,vsys_id FROM tsg_galaxy.gtpc_knowledge_base WHERE msg_type = 1 and uplink_teid !=0 %s order by last_update_time desc limit %s -#end - #sql("TABLE_INDEX_KEY") SELECT any(splitByString(', ',sorting_key)) AS index_key FROM tables_cluster where name = '%s_local' AND "database" = '%s' #end -#sql("UNSTRUCTURED_DATA_PATH") -SELECT log_id AS log_id,%s AS file_path FROM %s where %s AND notEmpty(%s) ORDER BY recv_time DESC LIMIT %s -#end - -#sql("UNSTRUCTURED_DATA_PATH_OLD") -SELECT common_log_id AS log_id,%s AS file_path FROM %s where %s AND notEmpty(%s) ORDER BY common_recv_time DESC LIMIT %s -#end - #sql("SQL_DATASETS") -SELECT id, identifier_name, category, execute_engine, type, template, description, generated_time, last_update_time FROM dataset %s ORDER BY last_update_time DESC +SELECT id, identifier_name, category, backend_engine, type, template, description, generated_time, last_update_time FROM dataset %s ORDER BY last_update_time DESC #end - -#sql("SQL_DATASETS_CATEGORY") -SELECT category FROM dataset GROUP BY category diff --git a/src/main/resources/job-sql-template.sql b/src/main/resources/job-sql-template.sql index 36ef3c40..5be4a538 100644 --- a/src/main/resources/job-sql-template.sql +++ b/src/main/resources/job-sql-template.sql @@ -17,13 +17,13 @@ SELECT count(*) AS logCount, %s AS totalMetric FROM %s %s LIMIT 1 UPSERT INTO "tsg_galaxy"."job_result"( ROWKEY, "detail"."is_done", "detail"."done_progress", "detail"."duration_time") VALUES('%s', %s, %s, %s) #end #sql("JOB_TOPK_DEFAULT") -SELECT %s AS value, %s(%s) AS %s FROM %s %s GROUP BY value ORDER BY %s DESC limit 10000 +SELECT %s AS value, %s(%s) AS %s%s FROM %s %s GROUP BY value ORDER BY %s%s DESC limit %s #end #sql("JOB_TOPK_ARRAY") -SELECT value, %s(metric) AS %s FROM (SELECT arrayJoin(items) AS value, metric FROM (SELECT %s AS items, %s AS metric FROM %s WHERE notEmpty(%s) %s GROUP BY items )) GROUP BY value ORDER BY %s DESC LIMIT 10000 +SELECT value, %s(metric) AS %s%s FROM (SELECT arrayJoin(items) AS value, metric FROM (SELECT IF(empty(%s), arrayPushBack(%s, NULL), %s) AS items, %s AS metric FROM %s %s GROUP BY items )) GROUP BY value ORDER BY %s%s DESC LIMIT %s #end #sql("JOB_TOPK_BIT") -SELECT value, %s(metric) AS %s FROM (SELECT arrayJoin(items) AS value, metric FROM (SELECT bitmaskToArray(%s) AS items, %s AS metric FROM %s %s GROUP BY items )) GROUP BY value ORDER BY %s DESC LIMIT 10000 +SELECT value, %s(metric) AS %s%s FROM (SELECT arrayJoin(items) AS value, metric FROM (SELECT bitmaskToArray(%s) AS items, %s AS metric FROM %s %s GROUP BY items )) GROUP BY value ORDER BY %s%s DESC LIMIT %s #end #sql("JOB_UPDATE_FIELD_VALUE_SINGLE_QUOTATION") UPSERT into "tsg_galaxy"."job_result" (ROWKEY, "%s"."%s") values('%s', '%s') @@ -50,7 +50,7 @@ SELECT %s FROM %s WHERE %s LIMIT 1 INSERT INTO saved_query_job (job_id, query_sql, state, done_progress, is_failed, result_message, elapsed, rows_read, bytes_read, result_rows, result_bytes, is_valid, start_time, end_time, last_update_time, generated_time ) VALUES ('%s', '%s', 'PENDING', 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, 1, NULL, NULL, '%s', '%s') #end #sql("SAVED_QUERY_JOB_STATUS") -SELECT state, done_progress, is_failed, is_valid, start_time, end_time, rows_read, query_sql, job_id FROM saved_query_job WHERE job_id IN ('%s') LIMIT %s +SELECT state, done_progress, is_failed, result_message, is_valid, start_time, end_time, rows_read, query_sql, job_id FROM saved_query_job WHERE job_id IN ('%s') LIMIT %s #end #sql("SAVED_QUERY_JOB_SAME") SELECT job_id AS id FROM saved_query_job WHERE is_valid = 1 AND state IN ( 'PENDING', 'RUNNING') AND is_failed = 0 AND query_sql = '%s' LIMIT 1 diff --git a/src/main/resources/schema-syntax-validation.json b/src/main/resources/schema-syntax-validation.json new file mode 100644 index 00000000..2870eba3 --- /dev/null +++ b/src/main/resources/schema-syntax-validation.json @@ -0,0 +1,120 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "type": { + "enum": ["record","enum","array","map","union","fixed"] + }, + "doc": { + "properties": { + "schema_query": { + "properties": { + "time": { + "items": { + "enum": [ + "$ENUM_VALUES$" + ] + }, + "uniqueItems": true + }, + "dimensions": { + "items": { + "enum": [ + "$ENUM_VALUES$" + ] + }, + "uniqueItems": true + }, + "metrics": { + "items": { + "enum": [ + "$ENUM_VALUES$" + ] + }, + "uniqueItems": true + }, + "filters": { + "items": { + "enum": [ + "$ENUM_VALUES$" + ] + }, + "uniqueItems": true + }, + "details": { + "patternProperties": { + ".*": { + "items": { + "enum": [ + "$ENUM_VALUES$" + ] + }, + "uniqueItems": true + } + } + } + } + }, + "decoded_as": { + "patternProperties": { + ".*": { + "patternProperties": { + ".*": { + "items": { + "enum": [ + "$ENUM_VALUES$" + ] + }, + "uniqueItems": true + } + } + } + } + } + }, + "message": { + "enum" : "{0}: described field not found in $.fields[*].name" + } + }, + "fields": { + "items": { + "properties": { + "type": { + "oneOf": [ + { + "type": "string", + "enum": ["null", "boolean", "int", "long", "float", "double", "bytes", "string"] + }, + { + "type": "object", + "properties": { + "type": { + "enum": ["array", "null", "boolean", "int", "long", "float", "double", "bytes", "string"] + }, + "items": { + "enum": ["null", "boolean", "int", "long", "float", "double", "bytes", "string"] + } + }, + "required": ["type"] + } + ] + }, + "doc": { + "required": [ + "visibility" + ] + } + }, + "required": [ + "name", + "type" + ] + } + } + }, + "required": [ + "type", + "name", + "namespace", + "fields" + ] +}
\ No newline at end of file diff --git a/src/test/java/com/mesalab/GalaxyQGWApplicationTests.java b/src/test/java/com/mesalab/GalaxyQGWApplicationTests.java index 276c3fa3..fd89b068 100644 --- a/src/test/java/com/mesalab/GalaxyQGWApplicationTests.java +++ b/src/test/java/com/mesalab/GalaxyQGWApplicationTests.java @@ -9,6 +9,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.web.server.LocalServerPort; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; @@ -18,12 +19,15 @@ import java.util.Map; import java.util.Objects; @RunWith(SpringRunner.class) -@SpringBootTest(classes = {GalaxyQGWApplication.class}, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) +@SpringBootTest(classes = {GalaxyQGWApplication.class}, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @EnableAutoConfiguration @ActiveProfiles("test") public class GalaxyQGWApplicationTests { private static final Log log = LogFactory.get(); + @LocalServerPort + protected int testPort; + static { System.setProperty("jasypt.encryptor.password", "galaxy"); System.setProperty("JM.SNAPSHOT.PATH", "config"); diff --git a/src/test/java/com/mesalab/knowledge/JsonSchemaTest.java b/src/test/java/com/mesalab/knowledge/JsonSchemaTest.java deleted file mode 100644 index ec78f262..00000000 --- a/src/test/java/com/mesalab/knowledge/JsonSchemaTest.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.mesalab.knowledge; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.fasterxml.jackson.databind.JsonNode; -import com.github.fge.jackson.JsonLoader; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.core.report.ProcessingMessage; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchema; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import com.mesalab.GalaxyQGWApplicationTests; -import org.apache.commons.lang3.Validate; -import org.junit.Test; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -import java.io.IOException; -import java.util.Iterator; - -/** - * @description: - * @author: zhq - * @create: 2020-07-31 - **/ -@EnableAutoConfiguration -public class JsonSchemaTest extends GalaxyQGWApplicationTests { - private static final Log log = LogFactory.get(); - - @Test - public void jsonTest() { - //创建jsonschema工厂 - String jsonStr = "{\n" + - " \"clientId\": null,\n" + - " \"query\": {\n" + - " \"queryType\": \"iplearning\",\n" + - " \"dataSource\": \"IP_LEARNING_VIEW\",\n" + - " \"parameters\": {\n" + - " \"match\": [\n" + - " {\n" + - " \"type\": \"substring\",\n" + - " \"fieldKey\": \"FQDN_NAME\",\n" + - " \"fieldValues\": \n" + - " [\"360\"]\n" + - " \n" + - " }\n" + - " ],\n" + - " \"range\": [\n" + - " {\n" + - " \"type\": \"ge\",\n" + - " \"fieldKey\": \"PROTOCOL\",\n" + - " \"fieldValues\": [\n" + - " \"HTTP\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"type\": \"eq\",\n" + - " \"fieldKey\": \"DEPTH\",\n" + - " \"fieldValues\": [\n" + - " 1\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"type\": \"ge\",\n" + - " \"fieldKey\": \"UNIQ_CIP\",\n" + - " \"fieldValues\": \n" + - " [5]\n" + - " \n" + - " }\n" + - " ],\n" + - " \"intervals\": [\n" + - " \"2020-07-01 00:00:00/2020-08-02 00:00:00\"]\n" + - " ,\n" + - " \"limit\": \"15\"\n" + - " }\n" + - " }\n" + - "}"; - try { - JsonSchemaFactory factory = JsonSchemaFactory.byDefault(); - //通过jsonschemaFactory获取jsonnode对象 - JsonNode schemaNode = JsonLoader.fromResource("/dsl-validation.json"); - //通过jsonstr字符串获取对应的jsonnode对象 - JsonNode dataNode = JsonLoader.fromString(jsonStr); - JsonSchema jsonSchema = factory.getJsonSchema(schemaNode); - //使用json-schema-validator中的jsonschema对象的validate方法对数据进行校验 - //获取处理的报告信息 - ProcessingReport processingReport = jsonSchema.validate(dataNode); - //获取完整的报告信息 - Iterator<ProcessingMessage> iterator = processingReport.iterator(); - StringBuffer sb = new StringBuffer(); - while (iterator.hasNext()) { - ProcessingMessage next = iterator.next(); - JsonNode jsonNode = next.asJson(); - sb.append("pointer on "); - sb.append(jsonNode.get("instance").get("pointer")); - sb.append(", "); - sb.append(next.getMessage()); - sb.append(". "); - } - //判断校验是否成功，如果为true成功 - Validate.isTrue(processingReport.isSuccess(), sb.toString()); - } catch (ProcessingException e) { - log.error("ProcessingException information: ", e.getMessage()); - } catch (IOException e) { - log.error("IOException information: ", e.getMessage()); - } - } -} diff --git a/src/test/java/com/mesalab/qgw/service/ApplicationAndProtocolTest.java b/src/test/java/com/mesalab/qgw/service/ApplicationAndProtocolTest.java new file mode 100644 index 00000000..dffa9959 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/ApplicationAndProtocolTest.java @@ -0,0 +1,113 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.context.request.async.DeferredResult; + +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class ApplicationAndProtocolTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + + @Test + public void testAppAndProtocolSummary() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_summary", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatusById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testAppAndProtocolTreeComposition() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_tree_composition", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResultById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testAppAndProtocolTreeThroughput() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_tree_throughput", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResult(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testAppAndProtocolTopApp() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_top_apps", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatus(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testAppAndProtocolTreeAppSummaryOneshot() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_app_summary", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 200); + } + + @Test + public void testAppAndProtocolAppRelatedInternalIpsOneshot() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_app_related_internal_ips", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 200); + } + + @Test + public void testAppAndProtocolAppThroughputOneshot() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_app_throughput", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 200); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/AutoPeriodDSLTest.java b/src/test/java/com/mesalab/qgw/service/AutoPeriodDSLTest.java new file mode 100644 index 00000000..cabe8c5a --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/AutoPeriodDSLTest.java @@ -0,0 +1,204 @@ +package com.mesalab.qgw.service; + +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.google.common.collect.Lists; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.utils.sqlparser.AutoPeriodHelper; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +/** + * TODO + * + * @Classname AutoPeriodTest + * @Date 2024/6/20 09:24 + * @Author wWei + */ +@EnableAutoConfiguration +public class AutoPeriodDSLTest extends GalaxyQGWApplicationTests { + private static final Log log = LogFactory.get(); + private static final DateTime now; + + static { + now = new DateTime(); + } + + @Test + public void testDSLLast5Minutes() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -5), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1S", granularity); + } + + @Test + public void testDSLLast5MinutesISO8601() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -5), "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1S", granularity); + } + + @Test + public void testDSLLast30Minutes() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -30), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT30S", granularity); + } + + @Test + public void testDSLLast1Hour() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -1), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT30S", granularity); + } + + @Test + public void testDSLLast6Hour() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -6), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1M", granularity); + } + + @Test + public void testDSLLast12Hour() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -12), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1M", granularity); + } + + @Test + public void testDSLLast1Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -1), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT5M", granularity); + } + + @Test + public void testDSLLast2Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -2), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT5M", granularity); + } + + @Test + public void testDSLLast3Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -3), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT5M", granularity); + } + + @Test + public void testDSLLast1Week() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetWeek(now, -1), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1H", granularity); + } + + @Test + public void testDSLLast30Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -30), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1H", granularity); + } + + @Test + public void testDSLLast365Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -365), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "P1D", granularity); + } + + @Test + public void testDSLConst() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_const", DSLQueryRequestParam.class); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1S", granularity); + } + + @Test + public void testDSLAutoRange() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto_const_range", DSLQueryRequestParam.class); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "P1D", granularity); + } + + @Test + public void testDSLTrafficSpectrumNetworkThroughputTrend() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "traffic_spectrum_network_throughput_trend_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -1), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1M", granularity); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/AutoPeriodSQLTest.java b/src/test/java/com/mesalab/qgw/service/AutoPeriodSQLTest.java new file mode 100644 index 00000000..214d0554 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/AutoPeriodSQLTest.java @@ -0,0 +1,227 @@ +package com.mesalab.qgw.service; + +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.utils.sqlparser.AutoPeriodHelper; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +/** + * TODO + * + * @Classname AutoPeriodTest + * @Date 2024/6/20 09:24 + * @Author wWei + */ +@EnableAutoConfiguration +public class AutoPeriodSQLTest extends GalaxyQGWApplicationTests { + private static final Log log = LogFactory.get(); + private static final DateTime now; + + static { + now = new DateTime(); + } + + private static final String originalSQL = "SELECT\n" + + " FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY('${start_time}', '${end_time}'), 'zero')) AS stat_time,\n" + + " AVG( in_pkts_per_sec ) AS avg_in_pkts_per_sec\n" + + "FROM\n" + + " (\n" + + " SELECT\n" + + " TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY('${start_time}', '${end_time}')) AS stat_time,\n" + + " RATE( in_pkts, SAMPLE_GRANULARITY('${start_time}', '${end_time}'), 1) AS in_pkts_per_sec\n" + + " FROM\n" + + " traffic_general_stat\n" + + " WHERE\n" + + " __time >= '${start_time}'\n" + + " AND __time < '${end_time}'\n" + + " AND vsys_id IN ( 1)\n" + + " AND ( 1 = 1 )\n" + + " GROUP BY\n" + + " TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY('${start_time}', '${end_time}'))\n" + + " HAVING TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY('${start_time}', '${end_time}')) >= '${start_time}'\n" + + " ORDER BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY('${start_time}', '${end_time}')) DESC\n" + + " )\n" + + "GROUP BY\n" + + " FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY('${start_time}', '${end_time}'), 'zero'))\n" + + "ORDER BY\n" + + " stat_time ASC\n" + + "LIMIT 10"; + + @Test + public void testSQLLast5Minutes() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -5), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: not contains PT1S", afterReplaceSQL.contains("PT1S")); + } + + @Test + public void testSQLLast5MinutesISO8601() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -5), "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: not contains PT1S", afterReplaceSQL.contains("PT1S")); + } + + @Test + public void testSQLLast30Minutes() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -30), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1S") && afterReplaceSQL.contains("PT30S")); + } + + @Test + public void testSQLLast1Hour() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -1), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1S") && afterReplaceSQL.contains("PT30S")); + } + + @Test + public void testSQLLast6Hour() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -6), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT30S") && afterReplaceSQL.contains("PT1M")); + } + + @Test + public void testSQLLast12Hour() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -6), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT30S") && afterReplaceSQL.contains("PT1M")); + } + + @Test + public void testSQLLast1Day() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -1), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1M") && afterReplaceSQL.contains("PT5M")); + } + + @Test + public void testSQLLast2Day() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -2), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1M") && afterReplaceSQL.contains("PT5M")); + } + + @Test + public void testSQLLast3Day() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -3), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1M") && afterReplaceSQL.contains("PT5M")); + } + + @Test + public void testSQLLast1Week() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetWeek(now, -1), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT5M") && afterReplaceSQL.contains("PT1H")); + } + + @Test + public void testSQLLast1Month() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -30), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT5M") && afterReplaceSQL.contains("PT1H")); + } + + @Test + public void testSQLLast365Day() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -365), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1H") && afterReplaceSQL.contains("P1D")); + } + + @Test + public void testSQLDrillDown() throws JSQLParserException { + String sql = "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY('${start_time}', '${end_time}'), 'zero')) AS \"Time\", device_group AS \"Device Group\", RATE(sessions, CHART_GRANULARITY('${start_time}', '${end_time}'), 1) AS \"Sessions\" FROM druid.statistics_rule WHERE __time >= '${start_time}' AND __time < '${end_time}' AND rule_id = 787368 AND template_id = 40010 AND chart_id = 44044 AND version = 1 AND device_group IN (SELECT device_group FROM statistics_rule WHERE __time >= '${start_time}' AND __time < '${end_time}' AND rule_id = 787368 AND template_id = 40010 AND chart_id = 44044 AND version = 1 AND statistics_rule.vsys_id IN (1) GROUP BY device_group ORDER BY RATE(sessions, CHART_GRANULARITY('${start_time}', '${end_time}'), 1) DESC LIMIT 10) AND statistics_rule.vsys_id IN (1) GROUP BY device_group, FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY('${start_time}', '${end_time}'), 'zero')) ORDER BY \"Time\" ASC LIMIT 65536"; + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -1), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = sql.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT30S")); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/AvroSchemaFormatTest.java b/src/test/java/com/mesalab/qgw/service/AvroSchemaFormatTest.java deleted file mode 100644 index efffa34d..00000000 --- a/src/test/java/com/mesalab/qgw/service/AvroSchemaFormatTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.mesalab.qgw.service; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import org.apache.http.HttpStatus; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -import static org.junit.Assert.assertEquals; - -@EnableAutoConfiguration -public class AvroSchemaFormatTest extends GalaxyQGWApplicationTests { - private static final Log log = LogFactory.get(); - @Autowired - public DiagnosisService diagnosisService; - @Before - public void testBefore() { - log.info("=================================================SCHEMA TEST BEGIN================================================="); - } - - @After - public void testAfter() { - log.info("=================================================SCHEMA TEST END================================================="); - } - - /** - * 测试schema - */ - @Test - public void testCheckSchema() { - BaseResult baseResult = diagnosisService.validateSchema(); - assertEquals("Schema format failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); - } - -} diff --git a/src/test/java/com/mesalab/qgw/service/ClickHouseTest.java b/src/test/java/com/mesalab/qgw/service/ClickHouseTest.java index 4e9dd11d..f6500835 100644 --- a/src/test/java/com/mesalab/qgw/service/ClickHouseTest.java +++ b/src/test/java/com/mesalab/qgw/service/ClickHouseTest.java @@ -8,7 +8,7 @@ import cn.hutool.log.LogFactory; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.enums.DBTypeEnum; +import com.mesalab.common.enums.DBEngineType; import com.mesalab.qgw.benchmark.DialectWriter; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Optional; //clickhouse-benchmark -i 93 --host 127.0.0.1 --port 9001 --user default --password "ceiec2019" --database tsg_galaxy_v3 < /root/session_record_queries_20210113203103.sql @@ -38,7 +37,7 @@ public class ClickHouseTest extends GalaxyQGWApplicationTests { @Test public void testSqlParser() { - String pocFile = dialectWriter.buildPocSQL(Lists.newLinkedList(), DBTypeEnum.CLICKHOUSE.getValue(), null); + String pocFile = dialectWriter.buildPocSQL(Lists.newLinkedList(), DBEngineType.CLICKHOUSE.getValue(), null); Optional<File> ckTestFile = Optional.of(new File(pocFile)); if (ckTestFile.isPresent()) { if (!ckTestFile.get().isFile()) { diff --git a/src/test/java/com/mesalab/qgw/service/DruidTest.java b/src/test/java/com/mesalab/qgw/service/DruidTest.java index a8b4fb51..55ffd62a 100644 --- a/src/test/java/com/mesalab/qgw/service/DruidTest.java +++ b/src/test/java/com/mesalab/qgw/service/DruidTest.java @@ -8,7 +8,7 @@ import cn.hutool.log.LogFactory; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.enums.DBTypeEnum; +import com.mesalab.common.enums.DBEngineType; import com.mesalab.qgw.benchmark.DialectWriter; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Optional; @@ -35,7 +34,7 @@ public class DruidTest extends GalaxyQGWApplicationTests { @Test public void testSqlParser() { - String pocFile = dialectWriter.buildPocSQL(Lists.newLinkedList(), DBTypeEnum.DRUID.getValue(), null); + String pocFile = dialectWriter.buildPocSQL(Lists.newLinkedList(), DBEngineType.DRUID.getValue(), null); Optional<File> ckTestFile = Optional.of(new File(pocFile)); if (ckTestFile.isPresent()) { if (!ckTestFile.get().isFile()) { diff --git a/src/test/java/com/mesalab/qgw/service/EngineTest.java b/src/test/java/com/mesalab/qgw/service/EngineTest.java index 65629173..b69d3e8a 100644 --- a/src/test/java/com/mesalab/qgw/service/EngineTest.java +++ b/src/test/java/com/mesalab/qgw/service/EngineTest.java @@ -4,7 +4,6 @@ import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.mesalab.GalaxyQGWApplicationTests; import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.enums.DiagnosisOptionEnum; import org.apache.http.HttpStatus; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -24,16 +23,5 @@ public class EngineTest extends GalaxyQGWApplicationTests { assertEquals("Schema validate failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); } - @Test - public void testMetadata() { - BaseResult baseResult = diagnosisService.validateMetadata(); - assertEquals("Metadata validate failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); - } - - @Test - public void testPocSql() { - BaseResult baseResult = diagnosisService.runPocSQL(false, DiagnosisOptionEnum.PARSE.getValue(), null); - assertEquals("POC sql parse failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); - } } diff --git a/src/test/java/com/mesalab/qgw/service/EntityTest.java b/src/test/java/com/mesalab/qgw/service/EntityTest.java deleted file mode 100644 index 2c5f498d..00000000 --- a/src/test/java/com/mesalab/qgw/service/EntityTest.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.service.EntityService; -import com.mesalab.services.service.RelationService; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -/** - * @Author wxs - * @Date 2022/9/28 - */ -@EnableAutoConfiguration -public class EntityTest extends GalaxyQGWApplicationTests { - - @Autowired - private EntityService entityService; - - @Autowired - private RelationService relationService; - - @Test - public void activeClientIp() { - - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "activeClientIp", ComDSLObject.class); - BaseResult entityInfo = entityService.getEntityInfo("activeclientip", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - - } - - @Test - public void topServerIp() { - - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "topServerIp", ComDSLObject.class); - BaseResult entityInfo = entityService.getEntityInfo("topserverip", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - - } - - @Test - public void topSni() { - - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "topSni", ComDSLObject.class); - BaseResult entityInfo = entityService.getEntityInfo("topsni", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - - } - - @Test - public void subScriberidPool() { - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "subScriberidPool", ComDSLObject.class); - BaseResult entityInfo = relationService.getRealRelation("subscriberidpool", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - } - - @Test - public void gtpc() { - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "gtpc", ComDSLObject.class); - BaseResult entityInfo = relationService.getRealRelation("gtpc", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - } -} diff --git a/src/test/java/com/mesalab/qgw/service/ExampleDataTest.java b/src/test/java/com/mesalab/qgw/service/ExampleDataTest.java new file mode 100644 index 00000000..c0f985c3 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/ExampleDataTest.java @@ -0,0 +1,62 @@ +package com.mesalab.qgw.service; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.DataTypeMapping; +import com.mesalab.common.utils.sqlparser.ExampleDataHelper; +import com.mesalab.common.utils.sqlparser.SelectItemHelper; +import com.mesalab.qgw.constant.DataTypeConst; +import com.mesalab.qgw.constant.MetaConst; +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.JSQLParserException; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +import java.util.List; +import java.util.Map; + + +@Slf4j +@EnableAutoConfiguration +public class ExampleDataTest extends GalaxyQGWApplicationTests { + + @Test + public void testSelectItemHelper() throws JSQLParserException { + String sql = "SELECT filed_1 AS \"Field A\", SUM(field_2) AS \"SUM(field_2)\", MAX(field_3) FROM table GROUP BY filed_1 ORDER BY \"Field A\" DESC LIMIT 10"; + Map<String, SelectItemHelper.AliasObject> selectItem = SelectItemHelper.getSelectItem(sql); + Assert.assertEquals("ok", 3, selectItem.size()); + } + + @Test + public void testExampleDataHelper() { + List<Map<String, String>> meta = initMeta(); + List<Object> objects = ExampleDataHelper.buildExampleData(ExampleDataHelper.QUERY_TYPE_GROUP_BY, meta, 1L, null, true); + Assert.assertEquals("ok", 1, objects.size()); + } + + private static List<Map<String, String>> initMeta() { + List<Map<String, String>> meta = Lists.newArrayList(); + Map<String, String> item1 = Maps.newHashMap(); + item1.put(MetaConst.META_NAME, "Field A"); + item1.put(MetaConst.META_TYPE, DataTypeMapping.STRING); + item1.put(MetaConst.META_CATEGORY, MetaConst.META_CATEGORY_DIMENSION); + item1.put(MetaConst.META_DATA_TYPE, DataTypeConst.IP); + meta.add(item1); + + Map<String, String> item2 = Maps.newHashMap(); + item2.put(MetaConst.META_NAME, "SUM(field_2)"); + item2.put(MetaConst.META_TYPE, DataTypeMapping.LONG); + item2.put(MetaConst.META_CATEGORY, MetaConst.META_CATEGORY_METRIC); + meta.add(item2); + + Map<String, String> item3 = Maps.newHashMap(); + item3.put(MetaConst.META_NAME, "MAX(field_3)"); + item3.put(MetaConst.META_TYPE, DataTypeMapping.LONG); + item3.put(MetaConst.META_CATEGORY, MetaConst.META_CATEGORY_METRIC); + meta.add(item3); + return meta; + } + +} diff --git a/src/test/java/com/mesalab/qgw/service/FieldDiscoveryTest.java b/src/test/java/com/mesalab/qgw/service/FieldDiscoveryTest.java new file mode 100644 index 00000000..db0ba405 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/FieldDiscoveryTest.java @@ -0,0 +1,114 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.context.request.async.DeferredResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class FieldDiscoveryTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + private static final List<String> fields = new ArrayList<>(); + + static { + fields.add("flags"); + fields.add("security_rule_list"); + fields.add("client_port"); + fields.add("server_ip"); + } + + @Test + public void testFieldDiscoveryDefault() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_default", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatusById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testFieldDiscoverySessions() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_sessions", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResultById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testFieldDiscoveryBytes() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_bytes", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResult(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testFieldDiscoveryIncomingBytes() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_incoming_bytes", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatus(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testFieldDiscoveryOutgoingBytes() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_outgoing_bytes", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/HttpClientServiceTest.java b/src/test/java/com/mesalab/qgw/service/HttpClientServiceTest.java new file mode 100644 index 00000000..d761f4fc --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/HttpClientServiceTest.java @@ -0,0 +1,24 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.qgw.service.impl.HttpClientServiceV2; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +@EnableAutoConfiguration +public class HttpClientServiceTest extends GalaxyQGWApplicationTests { + + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + @Autowired + private HttpClientServiceV2 httpClientServiceV2; + + @Test + public void testHttpGet() { + Assert.assertTrue(httpClientServiceV2.get("https://www.baidu.com", 1000).getStatusCode()==200); + Assert.assertTrue(httpClientServiceV2.get("http://www.baidu.com", 1).getStatusCode() == HttpStatusCodeEnum.GATEWAY_TIMEOUT.getCode()); + Assert.assertTrue(httpClientServiceV2.get("http://fake.www.baidu.com", 1000).getStatusCode() == HttpStatusCodeEnum.BAD_GATEWAY.getCode()); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/JobTest.java b/src/test/java/com/mesalab/qgw/service/JobTest.java deleted file mode 100644 index 4df5e0fd..00000000 --- a/src/test/java/com/mesalab/qgw/service/JobTest.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.controller.JobController; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.web.context.request.async.DeferredResult; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @Author wxs - * @Date 2022/9/28 - */ -@EnableAutoConfiguration -public class JobTest extends GalaxyQGWApplicationTests { - - @Autowired - private JobController jobController; - - @Test - public void fieldDiscoveryDefault() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "field_discovery_default", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - DeferredResult<BaseResult> baseResultDeferredResult = jobController.statisticsResult(id); - if(baseResultDeferredResult.hasResult()){ - BaseResult result = (BaseResult) baseResultDeferredResult.getResult(); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - } - - @Test - public void fieldDiscoveryBytes() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "field_discovery_bytes", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - DeferredResult<BaseResult> baseResultDeferredResult = jobController.statisticsResult(id); - if(baseResultDeferredResult.hasResult()){ - BaseResult result = (BaseResult) baseResultDeferredResult.getResult(); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - } - - @Test - public void longTerm() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "long_term", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - DeferredResult<BaseResult> baseResultDeferredResult = jobController.statisticsResult(id); - if(baseResultDeferredResult.hasResult()){ - BaseResult result = (BaseResult) baseResultDeferredResult.getResult(); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - - } - - @Test - public void report() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "report", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - BaseResult result = jobController.savedQueryStatus(id); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - BaseResult cancelled = jobController.cancelSavedQuery(id); - Assert.assertTrue(cancelled.getMessage(), cancelled.getStatus() == 202); - - } - - @Test - public void statistics() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "statistics_top", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - DeferredResult<BaseResult> baseResultDeferredResult = jobController.statisticsResult(id); - if(baseResultDeferredResult.hasResult()){ - BaseResult result = (BaseResult) baseResultDeferredResult.getResult(); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - } - -// @Test -// public void getResultReport() { -// BaseResult result = jobController.getResultReport(jobId); -// Assert.assertTrue(result.getMessage(), result.getStatus() == 200); -// } - -} diff --git a/src/test/java/com/mesalab/qgw/service/JsonSchemaValidatorTest.java b/src/test/java/com/mesalab/qgw/service/JsonSchemaValidatorTest.java new file mode 100644 index 00000000..a4352c20 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/JsonSchemaValidatorTest.java @@ -0,0 +1,86 @@ +package com.mesalab.qgw.service; + + +import cn.hutool.core.io.resource.ClassPathResource; +import cn.hutool.core.io.resource.Resource; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.utils.JsonSchemaValidator; +import com.mesalab.qgw.exception.QGWBusinessException; +import org.apache.commons.io.IOUtils; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.test.context.TestPropertySource; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; + + +@EnableAutoConfiguration +@TestPropertySource("classpath:") +public class JsonSchemaValidatorTest extends GalaxyQGWApplicationTests { + @Autowired + DatabaseService databaseService; + + @Test + public void testJsonSchema() { + try { + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + Map<String, Object> schemaMap = databaseService.getSchemaInfo("fields", "session_record", false); + boolean isValidSuccess = jsonSchemaValidator + .addRule("schema-syntax-validation.json", "schema") + .validateSchema(com.alibaba.fastjson2.JSON.toJSONString(schemaMap)); + Assert.assertTrue(isValidSuccess); + } catch (QGWBusinessException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testInvalidJsonSchema() { + boolean isValidError = true; + try { + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + Resource resource = new ClassPathResource("examples/invalidSessionRecordTest.json"); + String jsonStr = new String(IOUtils.toByteArray(resource.getStream()), StandardCharsets.UTF_8); + jsonSchemaValidator + .addRule("schema-syntax-validation.json", "schema") + .validateSchema(jsonStr); + } catch (Exception e) { + isValidError = false; + } + Assert.assertFalse(isValidError); + } + + @Test + public void testDSLRequest() { + try { + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + Resource resource = new ClassPathResource("examples/validDSLRequestTest.json"); + String jsonStr = new String(IOUtils.toByteArray(resource.getStream()), StandardCharsets.UTF_8); + boolean isValidSuccess = jsonSchemaValidator + .addRule("dsl-validation.json", "dsl") + .validateDSL(jsonStr); + Assert.assertTrue(isValidSuccess); + } catch (QGWBusinessException | IOException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testInvalidDSLRequest() { + boolean isValidError = true; + try { + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + Resource resource = new ClassPathResource("examples/invalidDSLRequestTest.json"); + String jsonStr = new String(IOUtils.toByteArray(resource.getStream()), StandardCharsets.UTF_8); + jsonSchemaValidator + .addRule("dsl-validation.json", "dsl") + .validateDSL(jsonStr); + } catch (Exception e) { + isValidError = false; + } + Assert.assertFalse(isValidError); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/KnowledgeBaseTest.java b/src/test/java/com/mesalab/qgw/service/KnowledgeBaseTest.java index 583f2e6f..2164e03b 100644 --- a/src/test/java/com/mesalab/qgw/service/KnowledgeBaseTest.java +++ b/src/test/java/com/mesalab/qgw/service/KnowledgeBaseTest.java @@ -2,10 +2,9 @@ package com.mesalab.qgw.service; import com.mesalab.GalaxyQGWApplicationTests; import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.entity.KnowledgeBase; +import com.mesalab.services.common.entity.KnowledgeBaseRequest; import com.mesalab.services.common.entity.KnowledgeConstant; -import com.mesalab.services.service.KnowledgeBaseService; -import org.apache.http.entity.ContentType; +import com.mesalab.services.service.KBService; import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -15,7 +14,6 @@ import org.springframework.web.multipart.MultipartFile; import java.io.File; import java.io.FileInputStream; -import java.util.Map; /** * @Author wxs @@ -23,58 +21,53 @@ import java.util.Map; */ @EnableAutoConfiguration public class KnowledgeBaseTest extends GalaxyQGWApplicationTests { + @Autowired - private KnowledgeBaseService knowledgeService; + private KBService kbService; + - private String id; @Test public void query() { - BaseResult result = knowledgeService.queryKnowledge(); + BaseResult result = kbService.getList(null, null); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); } @Test public void cycle() throws Exception { publish(); - update(); - toggle(); + updateFile(); + updateStatus(); delete(); + } private void publish() throws Exception { - KnowledgeBase knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "publishTest", KnowledgeBase.class); - File file = new File(KnowledgeConstant.DAT + File.separator + KnowledgeConstant.IP_BUILTIN + KnowledgeConstant.MMDB); + KnowledgeBaseRequest knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "publish", KnowledgeBaseRequest.class); + File file = new File(KnowledgeConstant.DAT + File.separator + KnowledgeConstant.ASN_BUILTIN + KnowledgeConstant.MMDB); FileInputStream inputStream = new FileInputStream(file); - MultipartFile multipartFile = new MockMultipartFile(file.getName(), file.getName(), - ContentType.APPLICATION_OCTET_STREAM.toString(), inputStream); - BaseResult result = knowledgeService.publishKnowledge(multipartFile, knowledgeBase); + MultipartFile multipartFile = new MockMultipartFile(file.getName(), inputStream); + BaseResult result = kbService.publishKnowledge(multipartFile, knowledgeBase); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - id = ((Map) result.getData()).get("id").toString(); } - private void update() throws Exception { - File file = new File(KnowledgeConstant.DAT + File.separator + KnowledgeConstant.IP_BUILTIN + KnowledgeConstant.MMDB); + private void updateFile() throws Exception { + KnowledgeBaseRequest knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "update", KnowledgeBaseRequest.class); + File file = new File(KnowledgeConstant.DAT + File.separator + KnowledgeConstant.ASN_BUILTIN + KnowledgeConstant.MMDB); FileInputStream inputStream = new FileInputStream(file); - MultipartFile multipartFile = new MockMultipartFile(file.getName(), file.getName(), - ContentType.APPLICATION_OCTET_STREAM.toString(), inputStream); - KnowledgeBase knowledgeBase = new KnowledgeBase(); - knowledgeBase.setId(id); - BaseResult result = knowledgeService.updateKnowledge(multipartFile, knowledgeBase); + MultipartFile multipartFile = new MockMultipartFile(file.getName(), inputStream); + BaseResult result = kbService.updateKnowledge(multipartFile, knowledgeBase); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); } - private void toggle() throws Exception{ - KnowledgeBase knowledgeBase = new KnowledgeBase(); - knowledgeBase.setId(id); - knowledgeBase.setIsValid(KnowledgeConstant.VALID); - BaseResult result = knowledgeService.toggleKnowledge(knowledgeBase); + private void updateStatus() { + KnowledgeBaseRequest knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "update_status", KnowledgeBaseRequest.class); + BaseResult result = kbService.updateStatus(knowledgeBase); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); } - private void delete() throws Exception { - KnowledgeBase knowledgeBase = new KnowledgeBase(); - knowledgeBase.setId(id); - BaseResult result = knowledgeService.deleteKnowledge(id); + private void delete() { + KnowledgeBaseRequest knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "delete", KnowledgeBaseRequest.class); + BaseResult result = kbService.deleteKnowledge(knowledgeBase); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); } } diff --git a/src/test/java/com/mesalab/qgw/service/MergeFunctionsHelperTest.java b/src/test/java/com/mesalab/qgw/service/MergeFunctionsHelperTest.java new file mode 100644 index 00000000..02a6ff6a --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/MergeFunctionsHelperTest.java @@ -0,0 +1,36 @@ +package com.mesalab.qgw.service; + +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.utils.sqlparser.FunctionsMergeHelper; +import net.sf.jsqlparser.JSQLParserException; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +/** + * TODO + * + * @Classname MergeFunctionsHelperTest + * @Date 2024/2/24 14:22 + * @Author wWei + */ +@EnableAutoConfiguration +public class MergeFunctionsHelperTest extends GalaxyQGWApplicationTests { + private static final Log log = LogFactory.get(); + + @Test + public void testPlainSelectSQL() throws JSQLParserException { + String originalSQL = "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero')) as a ,TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero') as b ,TIME_FLOOR_WITH_FILL(long_column, 'PT5M', 'zero') as c, SUM(bytes) AS bytes FROM table GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero')) ,TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero'),TIME_FLOOR_WITH_FILL(long_column, 'PT5M', 'zero') order by FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero')) asc ,TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero') desc ,TIME_FLOOR_WITH_FILL(long_column, 'PT5M', 'zero') desc, bytes asc, a desc limit 1"; + String build = FunctionsMergeHelper.build(originalSQL); + Assert.assertTrue(build.equals("SELECT TIME_FORMAT(TIME_FLOOR(column, 'PT5M'), 'yyyy-MM-dd HH:mm:ss') AS a, TIMESTAMP_TO_MILLIS(TIME_FLOOR(column, 'PT5M')) / 1000 AS b, TIMESTAMP_TO_MILLIS(TIME_FLOOR(MILLIS_TO_TIMESTAMP(long_column * 1000), 'PT5M')) / 1000 AS c, SUM(bytes) AS bytes FROM table GROUP BY TIME_FLOOR(column, 'PT5M'), TIME_FLOOR(column, 'PT5M'), TIME_FLOOR(MILLIS_TO_TIMESTAMP(long_column * 1000), 'PT5M') ORDER BY TIME_FLOOR(column, 'PT5M') ASC, TIME_FLOOR(column, 'PT5M') DESC, TIME_FLOOR(MILLIS_TO_TIMESTAMP(long_column * 1000), 'PT5M') DESC, bytes ASC, TIME_FLOOR(column, 'PT5M') DESC LIMIT 1")); + } + + @Test + public void testNestedSQL() throws JSQLParserException { + String originalSQL = "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( stat_time, 'PT5S','zero' )) AS stat_time, SUM(bytes) AS bytes FROM ( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1S') AS stat_time, SUM(in_bytes + out_bytes) AS bytes FROM statistics_rule WHERE __time >= '2024-02-01 00:00:00' and __time < '2024-02-02 00:00:00' and rule_id = 301 GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1S') ORDER BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1S') ASC ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( stat_time,'PT5S','zero' )) ORDER BY stat_time ASC, bytes DESC LIMIT 10"; + String build = FunctionsMergeHelper.build(originalSQL); + Assert.assertTrue(build.equals("SELECT TIME_FORMAT(TIME_FLOOR(MILLIS_TO_TIMESTAMP(stat_time * 1000), 'PT5S'), 'yyyy-MM-dd HH:mm:ss') AS stat_time, SUM(bytes) AS bytes FROM (SELECT TIMESTAMP_TO_MILLIS(TIME_FLOOR(__time, 'PT1S')) / 1000 AS stat_time, SUM(in_bytes + out_bytes) AS bytes FROM statistics_rule WHERE __time >= '2024-02-01 00:00:00' AND __time < '2024-02-02 00:00:00' AND rule_id = 301 GROUP BY TIME_FLOOR(__time, 'PT1S') ORDER BY TIME_FLOOR(__time, 'PT1S') ASC) GROUP BY TIME_FLOOR(MILLIS_TO_TIMESTAMP(stat_time * 1000), 'PT5S') ORDER BY TIME_FLOOR(MILLIS_TO_TIMESTAMP(stat_time * 1000), 'PT5S') ASC, bytes DESC LIMIT 10")); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/NetworkMonitorTest.java b/src/test/java/com/mesalab/qgw/service/NetworkMonitorTest.java deleted file mode 100644 index e74a4f9d..00000000 --- a/src/test/java/com/mesalab/qgw/service/NetworkMonitorTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.network.dsl.DSLObject; -import com.mesalab.network.service.NetworkMonitorService; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -/** - * TODO - * - * @Classname NetworkMonitorTest - * @Date 2022/12/7 09:45 - * @Author wWei - */ -@EnableAutoConfiguration -public class NetworkMonitorTest extends GalaxyQGWApplicationTests { - - @Autowired - NetworkMonitorService networkMonitorService; - - @Test - public void appDataSummary() { - DSLObject dslObject = jsonToInParameter("parameters/networkMonitorTest.json", "appDataSummary", DSLObject.class); - BaseResult result = networkMonitorService.buildAppData(dslObject); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - - @Test - public void internalIPDataSummary() { - DSLObject dslObject = jsonToInParameter("parameters/networkMonitorTest.json", "internalIPDataSummary", DSLObject.class); - BaseResult result = networkMonitorService.buildInternalIPData(dslObject); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - - @Test - public void appDataRateSummary() { - DSLObject dslObject = jsonToInParameter("parameters/networkMonitorTest.json", "appDataRateSummary", DSLObject.class); - BaseResult result = networkMonitorService.buildAppDataRate(dslObject); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - - @Test - public void appTrafficSummary() { - DSLObject dslObject = jsonToInParameter("parameters/networkMonitorTest.json", "appTrafficSummary", DSLObject.class); - BaseResult result = networkMonitorService.buildAppTraffic(dslObject); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } -} diff --git a/src/test/java/com/mesalab/qgw/service/ProtocolTreeTest.java b/src/test/java/com/mesalab/qgw/service/ProtocolTreeTest.java index 4121e9b5..76ed3ba8 100644 --- a/src/test/java/com/mesalab/qgw/service/ProtocolTreeTest.java +++ b/src/test/java/com/mesalab/qgw/service/ProtocolTreeTest.java @@ -6,7 +6,6 @@ import cn.hutool.log.LogFactory; import com.alibaba.fastjson2.JSON; import com.google.common.base.Stopwatch; import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.network.service.NetworkMonitorService; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -20,7 +19,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; public class ProtocolTreeTest extends GalaxyQGWApplicationTests { private static final Log log = LogFactory.get(); @Autowired - public NetworkMonitorService networkMonitorService; + public DSLService dslService; private List<Map> getLiveChartsData(String fileName) { File file = new File("benchmark" + File.separator + "live_traffic_chart" + File.separator + fileName); @@ -32,7 +31,7 @@ public class ProtocolTreeTest extends GalaxyQGWApplicationTests { @Test public void test5MinuteTreeOutput() { Stopwatch stopwatch = Stopwatch.createStarted(); - networkMonitorService.buildHierarchicalStructure(networkMonitorService.buildFlatStructure(getLiveChartsData("livecharts-5min.txt"))); + dslService.buildHierarchicalStructure(dslService.buildFlatStructure(getLiveChartsData("livecharts-5min.txt"))); stopwatch.stop(); long millis = stopwatch.elapsed(MILLISECONDS); log.info("time:{} ms", millis); @@ -41,7 +40,7 @@ public class ProtocolTreeTest extends GalaxyQGWApplicationTests { @Test public void test10MinuteTreeOutput() { Stopwatch stopwatch = Stopwatch.createStarted(); - networkMonitorService.buildHierarchicalStructure(networkMonitorService.buildFlatStructure(getLiveChartsData("livecharts-10min.txt"))); + dslService.buildHierarchicalStructure(dslService.buildFlatStructure(getLiveChartsData("livecharts-10min.txt"))); stopwatch.stop(); long millis = stopwatch.elapsed(MILLISECONDS); log.info("time:{} ms", millis); @@ -49,7 +48,7 @@ public class ProtocolTreeTest extends GalaxyQGWApplicationTests { @Test public void test30MinuteTreeOutput() { Stopwatch stopwatch = Stopwatch.createStarted(); - networkMonitorService.buildHierarchicalStructure(networkMonitorService.buildFlatStructure(getLiveChartsData("livecharts-30min.txt"))); + dslService.buildHierarchicalStructure(dslService.buildFlatStructure(getLiveChartsData("livecharts-30min.txt"))); stopwatch.stop(); long millis = stopwatch.elapsed(MILLISECONDS); log.info("time:{} ms", millis); @@ -57,7 +56,7 @@ public class ProtocolTreeTest extends GalaxyQGWApplicationTests { @Test public void test1HourTreeOutput() { Stopwatch stopwatch = Stopwatch.createStarted(); - networkMonitorService.buildHierarchicalStructure(networkMonitorService.buildFlatStructure(getLiveChartsData("livecharts-1h.txt"))); + dslService.buildHierarchicalStructure(dslService.buildFlatStructure(getLiveChartsData("livecharts-1h.txt"))); stopwatch.stop(); long millis = stopwatch.elapsed(MILLISECONDS); log.info("time:{} ms", millis); diff --git a/src/test/java/com/mesalab/qgw/service/RecommendTest.java b/src/test/java/com/mesalab/qgw/service/RecommendTest.java new file mode 100644 index 00000000..497e71e2 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/RecommendTest.java @@ -0,0 +1,48 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.context.request.async.DeferredResult; + +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class RecommendTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + + @Test + public void testIpLearningFqdnRelateIp() { + DSLQueryRequestParam body = jsonToInParameter("parameters/recommendTest.json", "ip_learning_fqdn_relate_ip", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatusById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testIpLearningActiveIpOneshot() { + DSLQueryRequestParam body = jsonToInParameter("parameters/recommendTest.json", "ip_learning_active_ip", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 200); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/SQLAdHocTest.java b/src/test/java/com/mesalab/qgw/service/SQLAdHocTest.java new file mode 100644 index 00000000..5f9083d9 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/SQLAdHocTest.java @@ -0,0 +1,123 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.SqlQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.context.request.async.DeferredResult; + +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class SQLAdHocTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + + @Test + public void testQuerySqlDefault() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_default", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatusById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQuerySqlOneshot() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_oneshot", SqlQueryRequestParam.class); + BaseResult baseResult = queryController.createSQLQuery(body); + Assert.assertTrue(baseResult.getMessage(), baseResult.getStatus() == 200); + } + + @Test + public void testQuerySqlNormal() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_normal", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatus(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQuerySqlBlocking() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_blocking", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResultById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQuerySqlJson() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_json", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResult(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQuerySqlCsv() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_csv", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResultById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQueryNoneSubQuery() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_oneshot_error_trigger_sub_query", SqlQueryRequestParam.class); + BaseResult baseResult = queryController.createSQLQuery(body); + Assert.assertTrue(baseResult.getMessage(), baseResult.getStatus() == 200); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/SQLSavedTest.java b/src/test/java/com/mesalab/qgw/service/SQLSavedTest.java new file mode 100644 index 00000000..c6048642 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/SQLSavedTest.java @@ -0,0 +1,41 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.SqlQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class SQLSavedTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + @Autowired + private QueryJobService queryJobService; + + @Test + public void testWholeProcess() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlSavedTest.json", "default", SqlQueryRequestParam.class); + + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + + BaseResult statusResult = queryJobService.getSavedQueryStatusById(id); + Assert.assertTrue(statusResult.getMessage(), statusResult.getStatus() == 200); + + BaseResult deletedResult = queryController.deleteJobById(id, 1); + Assert.assertTrue(deletedResult.getMessage(), deletedResult.getStatus() == 202); + + } +}
\ No newline at end of file diff --git a/src/test/java/com/mesalab/qgw/service/AvroSchemaDynamicTest.java b/src/test/java/com/mesalab/qgw/service/SchemaDynamicTest.java index 274e9487..1c7fb876 100644 --- a/src/test/java/com/mesalab/qgw/service/AvroSchemaDynamicTest.java +++ b/src/test/java/com/mesalab/qgw/service/SchemaDynamicTest.java @@ -14,14 +14,14 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import static org.junit.Assert.assertEquals; @EnableAutoConfiguration -public class AvroSchemaDynamicTest extends GalaxyQGWApplicationTests { +public class SchemaDynamicTest extends GalaxyQGWApplicationTests { private static final Log log = LogFactory.get(); @Autowired public DiagnosisService diagnosisService; @Test public void testValidateMetadata(){ - BaseResult baseResult = diagnosisService.validateMetadata(); - assertEquals("Validate metadata failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); + BaseResult baseResult = diagnosisService.validateSchema(); + assertEquals("Validate schema failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); } diff --git a/src/test/java/com/mesalab/qgw/service/TopEntityTest.java b/src/test/java/com/mesalab/qgw/service/TopEntityTest.java deleted file mode 100644 index 318cf863..00000000 --- a/src/test/java/com/mesalab/qgw/service/TopEntityTest.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.mesalab.qgw.service; - -import cn.hutool.core.io.FileUtil; -import cn.hutool.core.util.CharsetUtil; -import cn.hutool.core.util.NumberUtil; -import cn.hutool.core.util.StrUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.services.service.EntityService; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -import java.io.File; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -@EnableAutoConfiguration -public class TopEntityTest extends GalaxyQGWApplicationTests { - private static final Log log = LogFactory.get(); - - @Autowired - EntityService entityService; - - private List<Map<String, Object>> getEntityData(String fileName) { - File file = new File("benchmark" + File.separator + "entity_dataset" + File.separator + fileName); - List<String> list = FileUtil.readLines(file, CharsetUtil.UTF_8); - List<Map<String, Object>> data = Lists.newArrayList(); - for (String s : list) { - String[] split = StrUtil.split(s, "\t"); - Map<String, Object> map = new HashMap<>(16); - if (split.length >= 2) { - map.put("sessions", NumberUtil.isLong(split[1]) ? split[1] : "0"); - } - if (split.length >= 1) { - map.put("ssl_sni", split[0]); - data.add(map); - } - } - return data; - } - - @Test - public void testCUCCSNIOutput() { - Stopwatch stopwatch = Stopwatch.createStarted(); - entityService.mergeSNI(getEntityData("XJ-CUCC-SNI-Top200w.txt")); - stopwatch.stop(); - long millis = stopwatch.elapsed(MILLISECONDS); - log.info("time:{} ms", millis); - } - - @Test - public void testE21SNIOutput() { - Stopwatch stopwatch = Stopwatch.createStarted(); - entityService.mergeSNI(getEntityData("E21-SNI-Top200w.txt")); - stopwatch.stop(); - long millis = stopwatch.elapsed(MILLISECONDS); - log.info("time:{} ms", millis); - } - - @Test - public void testE21SNI20221011Output() { - Stopwatch stopwatch = Stopwatch.createStarted(); - entityService.mergeSNI(getEntityData("E21-SNI-Top3W-20221011.txt")); - stopwatch.stop(); - long millis = stopwatch.elapsed(MILLISECONDS); - log.info("time:{} ms", millis); - } - - @Test - public void testE21SNI20221020Output() { - Stopwatch stopwatch = Stopwatch.createStarted(); - entityService.mergeSNI(getEntityData("E21-SNI-Top120W-20221020.txt")); - stopwatch.stop(); - long millis = stopwatch.elapsed(MILLISECONDS); - log.info("time:{} ms", millis); - } - - - -} diff --git a/src/test/java/com/mesalab/qgw/service/UnstructuredTest.java b/src/test/java/com/mesalab/qgw/service/UnstructuredTest.java deleted file mode 100644 index 4285bc5a..00000000 --- a/src/test/java/com/mesalab/qgw/service/UnstructuredTest.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.service.UnstructuredService; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -@EnableAutoConfiguration -public class UnstructuredTest extends GalaxyQGWApplicationTests { - @Autowired - UnstructuredService unstructuredService; - - @Test - public void allFilePath(){ - ComDSLObject dslObject = jsonToInParameter("parameters/unstructuredTest.json", "all", ComDSLObject.class); - BaseResult UnstructuredData = unstructuredService.getUnstructuredData("all", dslObject); - Assert.assertTrue(UnstructuredData.getMessage(), UnstructuredData.getStatus() == 200); - } - - @Test - public void mailFilePath(){ - ComDSLObject dslObject = jsonToInParameter("parameters/unstructuredTest.json", "mail", ComDSLObject.class); - BaseResult UnstructuredData = unstructuredService.getUnstructuredData("mail", dslObject); - Assert.assertTrue(UnstructuredData.getMessage(), UnstructuredData.getStatus() == 200); - } - - @Test - public void pcapFilePath(){ - ComDSLObject dslObject = jsonToInParameter("parameters/unstructuredTest.json", "pcap", ComDSLObject.class); - BaseResult UnstructuredData = unstructuredService.getUnstructuredData("pcap", dslObject); - Assert.assertTrue(UnstructuredData.getMessage(), UnstructuredData.getStatus() == 200); - } - - @Test - public void httpFilePath(){ - ComDSLObject dslObject = jsonToInParameter("parameters/unstructuredTest.json", "http", ComDSLObject.class); - BaseResult UnstructuredData = unstructuredService.getUnstructuredData("http", dslObject); - Assert.assertTrue(UnstructuredData.getMessage(), UnstructuredData.getStatus() == 200); - } -} diff --git a/src/test/resources/examples/invalidDSLRequestTest.json b/src/test/resources/examples/invalidDSLRequestTest.json new file mode 100644 index 00000000..903f8f53 --- /dev/null +++ b/src/test/resources/examples/invalidDSLRequestTest.json @@ -0,0 +1,50 @@ +{ + "query": { + "parameters": { + "intervals": [ + "2024-03-14 00:00:00/2024-03-15 00:00:00" + ], + "limit": "3", + "match": [ + { + "fieldValues": [ + "SSL", + "HTTP", + "DNS" + ], + "type": "exactly" + }, + { + "fieldKey": "FQDN_NAME", + "fieldValues": [ + "itunes.apple", + "itunes.apple.com" + ], + "type": "exactly" + } + ], + "range": [ + { + "fieldKey": "VSYS_ID", + "type": "eq" + }, + { + "fieldKey": "DEPTH", + "fieldValues": [ + 1 + ], + "type": "eq" + }, + { + "fieldKey": "UNIQ_CIP", + "fieldValues": [ + 12 + ], + "type": "gt" + } + ], + "sort": [] + }, + "queryType": "iplearning" + } +}
\ No newline at end of file diff --git a/src/test/resources/examples/invalidSessionRecordTest.json b/src/test/resources/examples/invalidSessionRecordTest.json new file mode 100644 index 00000000..632ab4b0 --- /dev/null +++ b/src/test/resources/examples/invalidSessionRecordTest.json @@ -0,0 +1,6764 @@ +{ + "type": "record", + "name": "session_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "log_id", + "partition_key": "recv_time", + "index_key": [ + "vsys_id", + "security_action", + "proxy_action", + "decoded_as", + "data_center", + "device_group", + "recv_time" + ], + "ttl": 2592000, + "default_ttl": 2592000, + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)", + "metric_type": "counter", + "unit": "short" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "COUNT_DISTINCT(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)", + "metric_type": "counter", + "unit": "short" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "MEDIAN", + "label": "MEDIAN", + "function": "MEDIAN(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "MEDIAN_HDR", + "label": "MEDIAN_HDR", + "function": "MEDIAN_HDR(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "QUANTILE", + "label": "QUANTILE", + "function": "QUANTILE(expr,level)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P95_PERCENTILE", + "label": "P95_PERCENTILE", + "function": "QUANTILE(expr,0.95)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P99_PERCENTILE", + "label": "P99_PERCENTILE", + "function": "QUANTILE(expr,0.99)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "QUANTILE_HDR", + "label": "QUANTILE_HDR", + "function": "QUANTILE_HDR(expr,level)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P95_PERCENTILE_HDR", + "label": "P95_PERCENTILE_HDR", + "function": "QUANTILE_HDR(expr,0.95)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P99_PERCENTILE_HDR", + "label": "P99_PERCENTILE_HDR", + "function": "QUANTILE_HDR(expr,0.99)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "PERCENTILES_HDR", + "label": "PERCENTILES_HDR", + "function": "PERCENTILES_HDR(expr)", + "metric_type": "histogram", + "unit": "short" + }, + { + "name": "APPROX_COUNT_DISTINCT_HLLD", + "label": "COUNT_DISTINCT_HLLD", + "function": "APPROX_COUNT_DISTINCT_HLLD(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "APPROX_COUNT_DISTINCT_DS_HLL", + "label": "COUNT_DISTINCT_DS_HLL", + "function": "APPROX_COUNT_DISTINCT_DS_HLL(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "RATE", + "label": "RATE", + "function": "RATE(expr,duration)", + "metric_type": "gauge", + "unit": "sps" + }, + { + "name": "BITRATE", + "label": "BITRATE", + "function": "RATE(expr,duration)*8", + "metric_type": "gauge", + "unit": "bps" + } + ], + "date": [ + { + "name": "UNIX_TIMESTAMP", + "label": "UNIX_TIMESTAMP", + "function": "UNIX_TIMESTAMP(expr)" + }, + { + "name": "UNIX_TIMESTAMP_MILLIS", + "label": "UNIX_TIMESTAMP_MILLIS", + "function": "UNIX_TIMESTAMP_MILLIS(expr)" + }, + { + "name": "FROM_UNIXTIME", + "label": "FROM_UNIXTIME", + "function": "FROM_UNIXTIME(expr)" + }, + { + "name": "FROM_UNIXTIME_MILLIS", + "label": "FROM_UNIXTIME_MILLIS", + "function": "FROM_UNIXTIME_MILLIS(expr)" + }, + { + "name": "DATE_FORMAT", + "label": "DATE_FORMAT", + "function": "DATE_FORMAT(expr,format)" + }, + { + "name": "CONVERT_TZ", + "label": "CONVERT_TZ", + "function": "CONVERT_TZ(expr, from_tz, to_tz)" + }, + { + "name": "TIME_FLOOR_WITH_FILL", + "label": "TIME_FLOOR_WITH_FILL", + "function": "TIME_FLOOR_WITH_FILL(expr,period,fill)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + }, + { + "name": "bitAnd", + "label": "Bitwise AND", + "function": "bitAnd(expr, value)=value" + } + ] + }, + "schema_query": { + "time": [ + "recv_time", + "test", + "start_timestamp_ms", + "end_timestamp_ms", + "processing_time", + "ingestion_time", + "insert_time" + ], + "dimensions": [ + "session_id", + "session_id", + "device_id", + "out_link_id", + "in_link_id", + "data_center", + "sled_ip", + "device_group", + "address_type", + "decoded_as", + "vsys_id", + "flags", + "flags_identify_info", + "security_action", + "security_rule_list", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_action", + "proxy_rule_list", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "in_dest_mac", + "out_dest_mac", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "quic_sni", + "quic_version", + "quic_user_agent", + "ftp_account", + "ftp_url", + "ftp_link_type", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "security_rule_id", + "monitor_rule_id", + "proxy_rule_id", + "statistics_rule_id", + "shaping_rule_id", + "sc_rule_id" + ], + "metrics": [ + "client_ip", + "client_port", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "in_src_mac", + "out_src_mac", + "server_ip", + "server_port", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "in_dest_mac", + "out_dest_mac", + "app", + "decoded_path", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "duration_ms", + "tcp_handshake_latency_ms", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "http_url", + "http_host", + "http_user_agent", + "http_request_content_length", + "http_response_content_length", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_attachment_name", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qtype", + "dns_qclass", + "dns_qname", + "dns_cname", + "dns_response_latency_ms", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "quic_sni", + "quic_user_agent", + "ftp_account", + "ftp_url", + "sip_call_id", + "sip_server", + "ssh_hassh" + ], + "filters": [ + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "processing_time", + "ingestion_time", + "device_id", + "data_center", + "sled_ip", + "device_group", + "address_type", + "decoded_as", + "vsys_id", + "flags", + "flags_identify_info", + "security_action", + "security_rule_list", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "in_src_mac", + "out_src_mac", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "in_dest_mac", + "out_dest_mac", + "app_transition", + "app_debug_info", + "app", + "app_content", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qtype", + "dns_qclass", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "quic_sni", + "quic_version", + "quic_user_agent", + "ftp_account", + "ftp_url", + "ftp_link_type", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "security_rule_id", + "monitor_rule_id", + "proxy_rule_id", + "statistics_rule_id", + "shaping_rule_id", + "sc_rule_id" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "datetime", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "unix_timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "array", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "bit", + "functions": "COUNT,COUNT_DISTINCT" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "dateTime", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "unix_timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has,notEmpty,empty" + }, + { + "type": "bit", + "functions": "=,!=,bitAnd" + } + ] + }, + "details": { + "general": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info" + ], + "treatment": [ + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes" + ], + "source": [ + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number" + ], + "destination": [ + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain" + ], + "application": [ + "app_transition", + "app", + "app_debug_info", + "app_content", + "fqdn_category_list" + ], + "protocol": [ + "ip_protocol", + "decoded_path", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "ftp_account", + "ftp_url", + "ftp_link_type", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "quic_version", + "quic_sni", + "quic_user_agent", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe" + ], + "transmission": [ + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn" + ], + "other": [ + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc" + ] + } + }, + "data_view": { + "PROXY_INTERCEPT_EVENT_VIEW": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "security_rule_list", + "security_action", + "client_ip", + "client_port", + "server_fqdn", + "app", + "server_ip", + "server_port" + ] + } + }, + "decoded_as": { + "BASE": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "server_ip", + "server_port" + ] + }, + "HTTP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "http_url", + "server_ip", + "server_port" + ] + }, + "MAIL": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "mail_from", + "mail_to", + "mail_subject", + "server_ip", + "server_port" + ] + }, + "DNS": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "dns_qr", + "dns_qname", + "dns_qtype", + "server_ip", + "server_port" + ] + }, + "SSL": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "ssl_sni", + "server_ip", + "server_port" + ] + }, + "DTLS": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "dtls_sni", + "server_ip", + "server_port" + ] + }, + "QUIC": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "quic_version", + "quic_sni", + "quic_user_agent" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "quic_sni", + "server_ip", + "server_port" + ] + }, + "FTP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "ftp_account", + "ftp_url", + "ftp_link_type" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "ftp_url", + "server_ip", + "server_port" + ] + }, + "SIP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "server_ip", + "server_port" + ] + }, + "RTP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + }, + "RDP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "rdp_client_version", + "rdp_client_name" + ] + }, + "SSH": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "ssh_auth_success" + ] + }, + "Stratum": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_port", + "client_ip", + "server_ip", + "server_port", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program" + ] + } + }, + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "decoded_as", + "server_fqdn" + ], + "internal_columns": [ + "recv_time", + "log_id", + "flags_identify_info", + "encapsulation", + "app_debug_info", + "app_content", + "packet_capture_file", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_endpoint_a_ip", + "label": "Endpoint A IP", + "type": "string" + }, + { + "name": "gtp_endpoint_b_ip", + "label": "Endpoint B IP", + "type": "string" + }, + { + "name": "gtp_endpoint_a_port", + "label": "Endpoint A Port", + "type": "int" + }, + { + "name": "gtp_endpoint_b_port", + "label": "Endpoint B Port", + "type": "int" + }, + { + "name": "gtp_endpoint_a2b_teid", + "label": "Endpoint A2B TEID", + "type": "long" + }, + { + "name": "gtp_endpoint_b2a_teid", + "label": "Endpoint B2A TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label (c2s)", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label (s2c)", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction (c2s)", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction (s2c)", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC (c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC (c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC (s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC (s2c)", + "type": "string" + } + ], + "L2TP": [ + { + "name": "l2tp_version", + "label": "Version", + "type": "string" + }, + { + "name": "l2tp_lac2lns_tunnel_id", + "label": "LAC2LNS Tunnel ID", + "type": "int" + }, + { + "name": "l2tp_lns2lac_tunnel_id", + "label": "LNS2LAC Tunnel ID", + "type": "int" + }, + { + "name": "l2tp_lac2lns_session_id", + "label": "LAC2LNS Session ID", + "type": "int" + }, + { + "name": "l2tp_lns2lac_session_id", + "label": "LNS2LAC Session ID", + "type": "int" + }, + { + "name": "l2tp_access_concentrator_ip", + "label": "Access Concentrator IP", + "type": "string" + }, + { + "name": "l2tp_access_concentrator_port", + "label": "Access Concentrator Port", + "type": "int" + }, + { + "name": "l2tp_network_server_ip", + "label": "Network Server IP", + "type": "string" + }, + { + "name": "l2tp_network_server_port", + "label": "Network Server Port", + "type": "int" + } + ], + "PPTP": [ + { + "name": "pptp_uplink_tunnel_id", + "label": "UpLink Tunnel ID", + "type": "int" + }, + { + "name": "pptp_downlink_tunnel_id", + "label": "Down Tunnel ID", + "type": "int" + } + ], + "IPv4": [ + { + "name": "client_ip", + "label": "Client IP", + "type": "string" + }, + { + "name": "server_ip", + "label": "Server IP", + "type": "string" + } + ], + "IPv6": [ + { + "name": "client_ip", + "label": "Client IP", + "type": "string" + }, + { + "name": "server_ip", + "label": "Server IP", + "type": "string" + } + ] + }, + "measurements": { + "aggregated_metric_unit": { + "SUM": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes" + }, + "RATE": { + "sent_pkts": "pps", + "received_pkts": "pps", + "sent_bytes": "Bps", + "received_bytes": "Bps", + "tcp_c2s_lost_bytes": "Bps", + "tcp_s2c_lost_bytes": "Bps", + "tcp_c2s_o3_pkts": "pps", + "tcp_s2c_o3_pkts": "pps", + "tcp_c2s_rtx_pkts": "pps", + "tcp_s2c_rtx_pkts": "pps", + "tcp_c2s_rtx_bytes": "Bps", + "tcp_s2c_rtx_bytes": "Bps", + "http_request_content_length": "Bps", + "http_response_content_length": "Bps" + }, + "BITRATE": { + "sent_bytes": "bps", + "received_bytes": "bps", + "tcp_c2s_lost_bytes": "bps", + "tcp_s2c_lost_bytes": "bps", + "tcp_c2s_rtx_bytes": "bps", + "tcp_s2c_rtx_bytes": "bps", + "http_request_content_length": "bps", + "http_response_content_length": "bps" + }, + "MAX": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MIN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "AVG": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MEDIAN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P95_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P99_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + } + }, + "field_discovery_metric": { + "sessions": [ + { + "fn": "count", + "column": "log_id", + "value": "sessions", + "label": "Sessions", + "unit": "sessions" + } + ], + "bytes": [ + { + "fn": "sum", + "column": "sent_bytes + received_bytes", + "value": "bytes", + "label": "Bytes", + "unit": "bytes" + } + ], + "incoming_bytes": [ + { + "fn": "sum", + "column": "if(bitAnd(flags, 8) = 8, received_bytes, sent_bytes)", + "value": "incoming_bytes", + "label": "Incoming Bytes", + "unit": "bytes" + } + ], + "outgoing_bytes": [ + { + "fn": "sum", + "column": "if(bitAnd(flags, 8) = 8, sent_bytes, received_bytes)", + "value": "outgoing_bytes", + "label": "Outgoing Bytes", + "unit": "bytes" + } + ] + } + }, + "expression_fields": [ + { + "name": "internal_ip_list", + "label": "Internal IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_ip,''), if(bitAnd(flags, 16)=16, server_ip,''))" + } + }, + { + "name": "external_ip_list", + "label": "External IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_ip,''), if(bitAnd(flags, 16)!=16, server_ip,''))" + } + }, + { + "name": "security_rule_id", + "label": "Security Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(security_rule_list), arrayPushFront(security_rule_list, null),security_rule_list))" + } + }, + { + "name": "proxy_rule_id", + "label": "Proxy Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(proxy_rule_list), arrayPushFront(proxy_rule_list, null),proxy_rule_list))" + } + }, + { + "name": "monitor_rule_id", + "label": "Monitor Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(monitor_rule_list), arrayPushFront(monitor_rule_list, null),monitor_rule_list))" + } + }, + { + "name": "shaping_rule_id", + "label": "Shaping Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(shaping_rule_list), arrayPushFront(shaping_rule_list, null),shaping_rule_list))" + } + }, + { + "name": "sc_rule_id", + "label": "Service Chaining Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(sc_rule_list), arrayPushFront(sc_rule_list, null),sc_rule_list))" + } + }, + { + "name": "statistics_rule_id", + "label": "Statistics Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(statistics_rule_list), arrayPushFront(statistics_rule_list, null),statistics_rule_list))" + } + } + ], + "size": 0 + }, + "fields": [ + { + "name": "recv_time", + "type": { + "type": "long", + "logicalType": "unix_timestamp" + }, + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "label": "Receive Time" + }, + { + "name": "log_id", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Log ID" + }, + { + "name": "decoded_as", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "DTLS", + "value": "DTLS" + }, + { + "code": "QUIC", + "value": "QUIC" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "SSH", + "value": "SSH" + }, + { + "code": "Stratum", + "value": "Stratum" + }, + { + "code": "RDP", + "value": "RDP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + } + ], + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "label": "Decoded AS" + }, + { + "name": "session_id", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Session ID" + }, + { + "name": "start_timestamp_ms", + "type": { + "type": "string", + "logicalType": "datetime64" + }, + "doc": { + "constraints": { + "operator_functions": "=,!=,>,<,>=,<=", + "type": "datetime64" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Start Time" + }, + { + "name": "end_timestamp_ms", + "type": { + "type": "string", + "logicalType": "datetime64" + }, + "doc": { + "constraints": { + "operator_functions": "=,!=,>,<,>=,<=", + "type": "datetime64" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "End Time" + }, + { + "name": "duration_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Duration (ms)" + }, + { + "name": "tcp_handshake_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "TCP Handshake Latency (ms)" + }, + { + "name": "processing_time", + "type": { + "type": "long", + "logicalType": "unix_timestamp" + }, + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Processing Time" + }, + { + "name": "ingestion_time", + "type": { + "type": "long", + "logicalType": "unix_timestamp" + }, + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Ingestion Time" + }, + { + "name": "insert_time", + "type": { + "type": "long", + "logicalType": "unix_timestamp" + }, + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Insert Time" + }, + { + "name": "device_id", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Device ID" + }, + { + "name": "out_link_id", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Outgoing Link ID" + }, + { + "name": "in_link_id", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Incoming Link ID" + }, + { + "name": "device_tag", + "type": "string", + "doc": { + "visibility": "hidden", + "ttl": 2592000, + "size": 0 + }, + "label": "Device Tag" + }, + { + "name": "data_center", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [], + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "label": "Data Center" + }, + { + "name": "device_group", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + } + ], + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "label": "Device Group" + }, + { + "name": "sled_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Sled IP" + }, + { + "name": "address_type", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Address Type" + }, + { + "name": "vsys_id", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "default": 1, + "label": "Vsys ID" + }, + { + "name": "t_vsys_id", + "type": "int", + "doc": { + "allow_query": "false", + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Traffic Vsys ID" + }, + { + "name": "flags", + "type": { + "type": "long", + "logicalType": "bit" + }, + "doc": { + "constraints": { + "type": "bit", + "operator_functions": "=,!=,bitAnd" + }, + "data": [ + { + "code": "1", + "value": "Asymmetric" + }, + { + "code": "2", + "value": "Bulky" + }, + { + "code": "4", + "value": "CBR Streaming" + }, + { + "code": "8", + "value": "Client is Local" + }, + { + "code": "16", + "value": "Server is Local" + }, + { + "code": "32", + "value": "Download" + }, + { + "code": "64", + "value": "Interactive" + }, + { + "code": "128", + "value": "Inbound" + }, + { + "code": "256", + "value": "Outbound" + }, + { + "code": "512", + "value": "Pseudo Unidirectional" + }, + { + "code": "1024", + "value": "Streaming" + }, + { + "code": "2048", + "value": "Unidirectional" + }, + { + "code": "4096", + "value": "Random looking" + }, + { + "code": "8192", + "value": "C2S" + }, + { + "code": "16384", + "value": "S2C" + }, + { + "code": "32768", + "value": "Bidirectional" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Flags" + }, + { + "name": "flags_identify_info", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Flags Identify Info" + }, + { + "name": "security_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Security Rule List" + }, + { + "name": "security_action", + "type": "string", + "doc": { + "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "Deny", + "value": "Deny" + }, + { + "code": "Allow", + "value": "Allow" + } + ], + "ttl": null, + "size": 0 + }, + "label": "Security Action" + }, + { + "name": "monitor_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Monitor Rule List" + }, + { + "name": "sc_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Service Chaining Rule List" + }, + { + "name": "statistics_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Statistics Rule List" + }, + { + "name": "sc_rsp_raw", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Service Chaining Rendered Service Path (Raw)" + }, + { + "name": "sc_rsp_decrypted", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Service Chaining Rendered Service Path (Decrypted)" + }, + { + "name": "shaping_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Shaping Rule List" + }, + { + "name": "proxy_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Rule List" + }, + { + "name": "proxy_action", + "type": "string", + "doc": { + "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "Intercept", + "value": "Intercept" + }, + { + "code": "No Intercept", + "value": "No Intercept" + } + ], + "ttl": null, + "size": 0 + }, + "label": "Proxy Action" + }, + { + "name": "proxy_pinning_status", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "not pinning" + }, + { + "code": "1", + "value": "pinning" + }, + { + "code": "2", + "value": "maybe pinning" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Pinning Status" + }, + { + "name": "proxy_intercept_status", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "passthrough" + }, + { + "code": "1", + "value": "intercept" + }, + { + "code": "2", + "value": "shutdown" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Intercept Status" + }, + { + "name": "proxy_passthrough_reason", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Passthrough Reason" + }, + { + "name": "proxy_server_side_latency_ms", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Server-Side Latency (ms)" + }, + { + "name": "proxy_client_side_latency_ms", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Client-Side Latency (ms)" + }, + { + "name": "proxy_client_side_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Client-Side Version" + }, + { + "name": "proxy_server_side_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Server-Side Version" + }, + { + "name": "proxy_cert_verify", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Certificate Verify" + }, + { + "name": "proxy_intercept_error", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Intercept Error" + }, + { + "name": "monitor_mirrored_pkts", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Monitor Mirrored Packets" + }, + { + "name": "monitor_mirrored_bytes", + "type": "int", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Monitor Mirrored Bytes" + }, + { + "name": "client_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client IP" + }, + { + "name": "client_port", + "type": "int", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client Port" + }, + { + "name": "client_os_desc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client OS Description" + }, + { + "name": "client_geolocation", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client Geolocation" + }, + { + "name": "client_asn", + "type": "long", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client ASN" + }, + { + "name": "subscriber_id", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Subscriber ID" + }, + { + "name": "imei", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "IMEI" + }, + { + "name": "imsi", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "IMSI" + }, + { + "name": "apn", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "APN" + }, + { + "name": "phone_number", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Phone Number" + }, + { + "name": "server_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server IP" + }, + { + "name": "server_port", + "type": "int", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server Port" + }, + { + "name": "server_os_desc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server OS Description" + }, + { + "name": "server_geolocation", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server Geolocation" + }, + { + "name": "server_asn", + "type": "long", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server ASN" + }, + { + "name": "server_fqdn", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server FQDN" + }, + { + "name": "server_domain", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server Domain" + }, + { + "name": "app_transition", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Application Transition" + }, + { + "name": "app", + "type": "string", + "doc": { + "ttl": 2592000, + "size": 0 + }, + "label": "Application" + }, + { + "name": "app_debug_info", + "type": "string", + "doc": { + "visibility": "hidden", + "ttl": 2592000, + "size": 0 + }, + "label": "Application Debug Info" + }, + { + "name": "app_content", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Application Content" + }, + { + "name": "ip_protocol", + "type": "string", + "doc": { + "data": [ + { + "code": "tcp", + "value": "tcp" + }, + { + "code": "udp", + "value": "udp" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "IP Protocol" + }, + { + "name": "decoded_path", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Decoded Path" + }, + { + "name": "fqdn_category_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "dict_location": { + "path": "/v1/policy/object?type=fqdn_category", + "key": "category_id", + "value": "category_name" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "FQDN Category List" + }, + { + "name": "sent_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Packets Sent" + }, + { + "name": "received_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Packets Received" + }, + { + "name": "sent_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Bytes Sent" + }, + { + "name": "received_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Bytes Received" + }, + { + "name": "tcp_c2s_ip_fragments", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server IP Fragments" + }, + { + "name": "tcp_s2c_ip_fragments", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client IP Fragments" + }, + { + "name": "tcp_c2s_lost_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server Lost Bytes" + }, + { + "name": "tcp_s2c_lost_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client Lost Bytes" + }, + { + "name": "tcp_c2s_o3_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server Out-of-Order Packets" + }, + { + "name": "tcp_s2c_o3_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client Out-of-Order Packets" + }, + { + "name": "tcp_c2s_rtx_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server Retransmission Packets" + }, + { + "name": "tcp_s2c_rtx_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client Retransmission Packets" + }, + { + "name": "tcp_c2s_rtx_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server Retransmission Bytes" + }, + { + "name": "tcp_s2c_rtx_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client Retransmission Bytes" + }, + { + "name": "tcp_rtt_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Round-trip Time (ms)" + }, + { + "name": "tcp_client_isn", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client ISN" + }, + { + "name": "tcp_server_isn", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server ISN" + }, + { + "name": "packet_capture_file", + "type": "string", + "doc": { + "allow_query": "false", + "visibility": "enabled", + "constraints": { + "type": "file" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "Packet Capture File" + }, + { + "name": "in_src_mac", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Incoming Source MAC" + }, + { + "name": "out_src_mac", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Outgoing Source MAC" + }, + { + "name": "in_dest_mac", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Incoming Destination MAC" + }, + { + "name": "out_dest_mac", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Outgoing Destination MAC" + }, + { + "name": "encapsulation", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Encapsulation" + }, + { + "name": "dup_traffic_flag", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Duplicate Traffic Flag" + }, + { + "name": "tunnel_endpoint_a_desc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Tunnel Endpoint A Description" + }, + { + "name": "tunnel_endpoint_b_desc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Tunnel Endpoint B Description" + }, + { + "name": "http_url", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.URL" + }, + { + "name": "http_host", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Host" + }, + { + "name": "http_request_line", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Request Line" + }, + { + "name": "http_response_line", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Line" + }, + { + "name": "http_request_content_length", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Request Content-Length" + }, + { + "name": "http_request_content_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Request Content-Type" + }, + { + "name": "http_response_content_length", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Content-Length" + }, + { + "name": "http_response_content_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Content Type" + }, + { + "name": "http_request_body", + "type": "string", + "doc": { + "allow_query": "false", + "constraints": { + "type": "file" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Request Body" + }, + { + "name": "http_response_body", + "type": "string", + "doc": { + "allow_query": "false", + "constraints": { + "type": "file" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Body" + }, + { + "name": "http_proxy_flag", + "type": "int", + "doc": { + "visibility": "hidden", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Proxy Flag" + }, + { + "name": "http_sequence", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Sequence" + }, + { + "name": "http_cookie", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Cookie" + }, + { + "name": "http_referer", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Referer" + }, + { + "name": "http_user_agent", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.User-Agent" + }, + { + "name": "http_set_cookie", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Set-Cookie" + }, + { + "name": "http_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Version" + }, + { + "name": "http_status_code", + "type": "int", + "doc": { + "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Status Code" + }, + { + "name": "http_response_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Latency (ms)" + }, + { + "name": "http_action_file_size", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Action File Size" + }, + { + "name": "http_session_duration_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Session Duration (ms)" + }, + { + "name": "mail_protocol_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Protocol Type" + }, + { + "name": "mail_account", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Account" + }, + { + "name": "mail_from_cmd", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.From CMD" + }, + { + "name": "mail_to_cmd", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.To CMD" + }, + { + "name": "mail_from", + "type": "string", + "doc": { + "constraints": { + "type": "email" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.From" + }, + { + "name": "mail_password", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Password" + }, + { + "name": "mail_to", + "type": "string", + "doc": { + "constraints": { + "type": "email" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.To" + }, + { + "name": "mail_cc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.CC" + }, + { + "name": "mail_bcc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.BCC" + }, + { + "name": "mail_subject", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Subject" + }, + { + "name": "mail_subject_charset", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Subject Charset" + }, + { + "name": "mail_attachment_name", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Attachment Name" + }, + { + "name": "mail_attachment_name_charset", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Attachment Name Charset" + }, + { + "name": "mail_eml_file", + "type": "string", + "doc": { + "constraints": { + "type": "file" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.EML File" + }, + { + "name": "dns_message_id", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.Message ID" + }, + { + "name": "dns_qr", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QR" + }, + { + "name": "dns_opcode", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.OPCODE" + }, + { + "name": "dns_aa", + "type": "int", + "doc": { + "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.AA" + }, + { + "name": "dns_tc", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.TC" + }, + { + "name": "dns_rd", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.RD" + }, + { + "name": "dns_ra", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.RA" + }, + { + "name": "dns_rcode", + "type": "int", + "doc": { + "data": [ + { + "code": 0, + "value": "NoError" + }, + { + "code": 1, + "value": "FormErr" + }, + { + "code": 2, + "value": "ServFail" + }, + { + "code": 3, + "value": "NXDomain" + }, + { + "code": 4, + "value": "NotImp" + }, + { + "code": 5, + "value": "Refused" + }, + { + "code": 6, + "value": "YXDomain" + }, + { + "code": 7, + "value": "YXRRSet" + }, + { + "code": 8, + "value": "NXRRSet" + }, + { + "code": 9, + "value": "NotAuth" + }, + { + "code": 10, + "value": "NotZone" + }, + { + "code": 16, + "value": "BADSIG" + }, + { + "code": 17, + "value": "BADKEY" + }, + { + "code": 18, + "value": "BADTIME" + }, + { + "code": 19, + "value": "BADMODE" + }, + { + "code": 20, + "value": "BADNAME" + }, + { + "code": 21, + "value": "BADALG" + } + ], + "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.RCODE" + }, + { + "name": "dns_qdcount", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QDCOUNT" + }, + { + "name": "dns_ancount", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.ANCOUNT" + }, + { + "name": "dns_nscount", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.NSCOUNT" + }, + { + "name": "dns_arcount", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.ARCOUNT" + }, + { + "name": "dns_qname", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QNAME" + }, + { + "name": "dns_qtype", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "3", + "value": "MD" + }, + { + "code": "4", + "value": "MF" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "7", + "value": "MB" + }, + { + "code": "8", + "value": "MG" + }, + { + "code": "9", + "value": "MR" + }, + { + "code": "10", + "value": "NULL" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "14", + "value": "MINFO" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "16", + "value": "TXT" + }, + { + "code": "17", + "value": "RP" + }, + { + "code": "18", + "value": "AFSDB" + }, + { + "code": "19", + "value": "X25" + }, + { + "code": "20", + "value": "ISDN" + }, + { + "code": "21", + "value": "RT" + }, + { + "code": "22", + "value": "NSAP" + }, + { + "code": "23", + "value": "NSAP" + }, + { + "code": "24", + "value": "SIG" + }, + { + "code": "25", + "value": "KEY" + }, + { + "code": "26", + "value": "PX" + }, + { + "code": "27", + "value": "GPOS" + }, + { + "code": "28", + "value": "AAAA" + }, + { + "code": "29", + "value": "LOC" + }, + { + "code": "30", + "value": "EID" + }, + { + "code": "31", + "value": "NIMLOC" + }, + { + "code": "32", + "value": "NB" + }, + { + "code": "33", + "value": "SRV" + }, + { + "code": "34", + "value": "ATMA" + }, + { + "code": "35", + "value": "NAPTR" + }, + { + "code": "36", + "value": "KX" + }, + { + "code": "37", + "value": "CERT" + }, + { + "code": "38", + "value": "A6" + }, + { + "code": "39", + "value": "DNAME" + }, + { + "code": "40", + "value": "SINK" + }, + { + "code": "41", + "value": "OPT" + }, + { + "code": "42", + "value": "APL" + }, + { + "code": "43", + "value": "DS" + }, + { + "code": "44", + "value": "SSHFP" + }, + { + "code": "45", + "value": "IPSECKEY" + }, + { + "code": "46", + "value": "RRSIG" + }, + { + "code": "47", + "value": "NSEC" + }, + { + "code": "48", + "value": "DNSKEY" + }, + { + "code": "49", + "value": "DHCID" + }, + { + "code": "50", + "value": "NSEC3" + }, + { + "code": "51", + "value": "NSEC3PARAM" + }, + { + "code": "52", + "value": "TLSA" + }, + { + "code": "53", + "value": "SMIMEA" + }, + { + "code": "55", + "value": "HIP" + }, + { + "code": "59", + "value": "CDS" + }, + { + "code": "60", + "value": "CDNSKEY" + }, + { + "code": "61", + "value": "OPENPGPKEY" + }, + { + "code": "62", + "value": "CSYNC" + }, + { + "code": "63", + "value": "ZONEMD" + }, + { + "code": "64", + "value": "SVCB" + }, + { + "code": "65", + "value": "HTTPS" + }, + { + "code": "99", + "value": "SPF" + }, + { + "code": "100", + "value": "UINFO" + }, + { + "code": "101", + "value": "UID" + }, + { + "code": "102", + "value": "GID" + }, + { + "code": "103", + "value": "UNSPEC" + }, + { + "code": "108", + "value": "EUI48" + }, + { + "code": "109", + "value": "EUI64" + }, + { + "code": "249", + "value": "TKEY" + }, + { + "code": "250", + "value": "TSIG" + }, + { + "code": "251", + "value": "IXFR" + }, + { + "code": "252", + "value": "AXFR" + }, + { + "code": "253", + "value": "MAILB" + }, + { + "code": "254", + "value": "MAILA" + }, + { + "code": "255", + "value": "*" + }, + { + "code": "256", + "value": "URI" + }, + { + "code": "257", + "value": "CAA" + }, + { + "code": "32768", + "value": "TA" + }, + { + "code": "32769", + "value": "DLV" + }, + { + "code": "65521", + "value": "INTEGRITY" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QTYPE" + }, + { + "name": "dns_qclass", + "type": "int", + "doc": { + "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QCLASS" + }, + { + "name": "dns_cname", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.CNAME" + }, + { + "name": "dns_sub", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.SUB" + }, + { + "name": "dns_rr", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.RR" + }, + { + "name": "dns_response_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.Response Latency (ms)" + }, + { + "name": "ssl_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.Version" + }, + { + "name": "ssl_sni", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.SNI" + }, + { + "name": "ssl_san", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.SAN" + }, + { + "name": "ssl_cn", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.CN" + }, + { + "name": "ssl_handshake_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.Handshake Latency (ms)" + }, + { + "name": "ssl_ja3_hash", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.JA3 Hash" + }, + { + "name": "ssl_ja3s_hash", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.JA3S Hash" + }, + { + "name": "ssl_cert_issuer", + "type": "string", + "doc": { + "constraints": { + "type": "items" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.Issuer" + }, + { + "name": "ssl_cert_subject", + "type": "string", + "doc": { + "constraints": { + "type": "items" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.Subject" + }, + { + "name": "ssl_esni_flag", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.ESNI Flag" + }, + { + "name": "ssl_ech_flag", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": " SSL.ECH Flag" + }, + { + "name": "dtls_cookie", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Cookie" + }, + { + "name": "dtls_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Version" + }, + { + "name": "dtls_sni", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.SNI" + }, + { + "name": "dtls_san", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.SAN" + }, + { + "name": "dtls_cn", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.CN" + }, + { + "name": "dtls_handshake_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Handshake Latency (ms)" + }, + { + "name": "dtls_ja3_fingerprint", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.JA3 Fingerprint" + }, + { + "name": "dtls_ja3_hash", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.JA3 Hash" + }, + { + "name": "dtls_cert_issuer", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Certificate Issuer" + }, + { + "name": "dtls_cert_subject", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Certificate Subject" + }, + { + "name": "quic_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "QUIC.Version" + }, + { + "name": "quic_sni", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "QUIC.SNI" + }, + { + "name": "quic_user_agent", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "QUIC.User-Agent" + }, + { + "name": "ftp_account", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "FTP.Account" + }, + { + "name": "ftp_url", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "FTP.URL" + }, + { + "name": "ftp_link_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "FTP.Link Type" + }, + { + "name": "sip_call_id", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Call-ID" + }, + { + "name": "sip_originator_description", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator" + }, + { + "name": "sip_responder_description", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder" + }, + { + "name": "sip_user_agent", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.User-Agent" + }, + { + "name": "sip_server", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Server" + }, + { + "name": "sip_originator_sdp_connect_ip", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator IP" + }, + { + "name": "sip_originator_sdp_media_port", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator Port" + }, + { + "name": "sip_originator_sdp_media_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator Media Type" + }, + { + "name": "sip_originator_sdp_content", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator Content" + }, + { + "name": "sip_responder_sdp_connect_ip", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder IP" + }, + { + "name": "sip_responder_sdp_media_port", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder Port" + }, + { + "name": "sip_responder_sdp_media_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder Media Type" + }, + { + "name": "sip_responder_sdp_content", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder Content" + }, + { + "name": "sip_duration_s", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Duration (s)" + }, + { + "name": "sip_bye", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Bye" + }, + { + "name": "rtp_payload_type_c2s", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RTP.Payload Type (C2S)" + }, + { + "name": "rtp_payload_type_s2c", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RTP.Payload Type (S2C)" + }, + { + "name": "rtp_pcap_path", + "type": "string", + "doc": { + "allow_query": "false", + "constraints": { + "type": "file" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RTP.PCAP" + }, + { + "name": "rtp_originator_dir", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RTP.Direction" + }, + { + "name": "ssh_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Version" + }, + { + "name": "ssh_auth_success", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Authentication Result" + }, + { + "name": "ssh_client_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Client Version" + }, + { + "name": "ssh_server_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Server Version" + }, + { + "name": "ssh_cipher_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Encryption Algorithm" + }, + { + "name": "ssh_mac_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Signing Algorithm" + }, + { + "name": "ssh_compression_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Compression Algorithm" + }, + { + "name": "ssh_kex_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Key Exchange Algorithm" + }, + { + "name": "ssh_host_key_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Server Host Key Algorithm" + }, + { + "name": "ssh_host_key", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Server Key Fingerprint" + }, + { + "name": "ssh_hassh", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.HASSH" + }, + { + "name": "stratum_cryptocurrency", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Stratum.Cryptocurrency" + }, + { + "name": "stratum_mining_pools", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Stratum.Mining Pools" + }, + { + "name": "stratum_mining_program", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Stratum.Mining Program" + }, + { + "name": "stratum_mining_subscribe", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Stratum.Mining Subscribe" + }, + { + "name": "rdp_cookie", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Cookie" + }, + { + "name": "rdp_security_protocol", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Security Protocol" + }, + { + "name": "rdp_client_channels", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Client Channels" + }, + { + "name": "rdp_keyboard_layout", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Keyboard Layout" + }, + { + "name": "rdp_client_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Client Version" + }, + { + "name": "rdp_client_name", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Client Name" + }, + { + "name": "rdp_client_product_id", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Client Product ID" + }, + { + "name": "rdp_desktop_width", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Desktop Width" + }, + { + "name": "rdp_desktop_height", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Desktop Height" + }, + { + "name": "rdp_requested_color_depth", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Requested Color Depth" + }, + { + "name": "rdp_certificate_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Certificate Type" + }, + { + "name": "rdp_certificate_count", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Certificate Count" + }, + { + "name": "rdp_certificate_permanent", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Certificate Permanent" + }, + { + "name": "rdp_encryption_level", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Encryption Level" + }, + { + "name": "rdp_encryption_method", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Encryption Method" + } + ] +} diff --git a/src/test/resources/examples/validDSLRequestTest.json b/src/test/resources/examples/validDSLRequestTest.json new file mode 100644 index 00000000..c7e2225c --- /dev/null +++ b/src/test/resources/examples/validDSLRequestTest.json @@ -0,0 +1,55 @@ +{ + "query": { + "dataSource": "IP_LEARNING_VIEW", + "parameters": { + "intervals": [ + "2024-03-14 00:00:00/2024-03-15 00:00:00" + ], + "limit": "3", + "match": [ + { + "fieldKey": "PROTOCOL", + "fieldValues": [ + "SSL", + "HTTP", + "DNS" + ], + "type": "exactly" + }, + { + "fieldKey": "FQDN_NAME", + "fieldValues": [ + "itunes.apple", + "itunes.apple.com" + ], + "type": "exactly" + } + ], + "range": [ + { + "fieldKey": "VSYS_ID", + "fieldValues": [ + 1 + ], + "type": "eq" + }, + { + "fieldKey": "DEPTH", + "fieldValues": [ + 1 + ], + "type": "eq" + }, + { + "fieldKey": "UNIQ_CIP", + "fieldValues": [ + 12 + ], + "type": "gt" + } + ], + "sort": [] + }, + "queryType": "iplearning" + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/applicationAndProtocolTest.json b/src/test/resources/parameters/applicationAndProtocolTest.json new file mode 100644 index 00000000..2a8c043d --- /dev/null +++ b/src/test/resources/parameters/applicationAndProtocolTest.json @@ -0,0 +1,60 @@ +{ + "application_and_protocol_summary": { + "name": "application-and-protocol-summary", + "granularity": "PT5S", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ] + }, + "application_and_protocol_tree_composition": { + "name": "application-and-protocol-tree-composition", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30T00:00:00+08:00/2024-01-31T00:00:00+08:00" + ] + }, + "application_and_protocol_tree_throughput": { + "name": "application-and-protocol-tree-throughput", + "granularity": "PT1H", + "filter": " (vsys_id = 1) AND (protocol_stack_id = 'ETHERNET.IPv4' OR ( protocol_stack_id LIKE 'ETHERNET.IPv4.%' AND NOT CONTAINS_STRING(REPLACE(protocol_stack_id, 'ETHERNET.IPv4.', ''), '.')))", + "intervals": [ + "2024-01-30T00:00:00.000+01:00/2024-01-31T00:00:00.000+01:00" + ] + }, + "application_and_protocol_top_apps": { + "name": "application-and-protocol-top-apps", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "limit": 10 + }, + "application_and_protocol_app_summary": { + "name": "application-and-protocol-app-summary", + "execution_mode":"oneshot", + "filter": " vsys_id = 1 AND app_name IN ('ftp', 'http')", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ] + }, + "application_and_protocol_app_related_internal_ips": { + "name": "application-and-protocol-app-related-internal-ips", + "execution_mode":"oneshot", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "limit": 10 + }, + "application_and_protocol_app_throughput": { + "name": "application-and-protocol-app-throughput", + "execution_mode":"oneshot", + "granularity": "PT15S", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "limit": 10 + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/dslAutoGranularityTest.json b/src/test/resources/parameters/dslAutoGranularityTest.json new file mode 100644 index 00000000..2f8a9d4f --- /dev/null +++ b/src/test/resources/parameters/dslAutoGranularityTest.json @@ -0,0 +1,27 @@ +{ + "application_and_protocol_summary_auto": { + "name": "application-and-protocol-summary", + "filter": "vsys_id = 1" + }, + "application_and_protocol_summary_const": { + "name": "application-and-protocol-summary", + "granularity": "PT1S", + "filter": "vsys_id = 1", + "interval": [ + "2019-01-01 00:00:00/2019-10-01 00:00:10" + ] + }, + "application_and_protocol_summary_auto_const_range": { + "name": "application-and-protocol-summary", + "granularity": "CHART_GRANULARITY('2019-01-01 00:00:00', '2019-10-01 00:00:10')", + "filter": "vsys_id = 1", + "interval": [ + "2019-01-01 00:00:00/2019-10-01 00:00:10" + ] + }, + "traffic_spectrum_network_throughput_trend_auto": { + "name": "traffic-spectrum-network-throughput-trend", + "filter": "vsys_id in (1) ", + "execution_mode": "oneshot" + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/entityTest.json b/src/test/resources/parameters/entityTest.json deleted file mode 100644 index 5a94461e..00000000 --- a/src/test/resources/parameters/entityTest.json +++ /dev/null @@ -1,132 +0,0 @@ -{ - "activeClientIp": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "session_record", - "limit": "10000", - "parameters": { - "match": [ - { - "type": "exactly", - "fieldKey": "app", - "fieldValues": [ - "Freegate" - ] - } - ], - "range": [ - { - "type": "eq", - "fieldKey": "vsys_id", - "fieldValues": [ - 1 - ] - } - ], - "intervals": [ - "2020-08-15T00:00:00.865Z/2022-08-15T00:30:00.865Z" - ] - } - } - }, - "topServerIp": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "session_record", - "limit": "10000", - "parameters": { - "range": [ - { - "type": "eq", - "fieldKey": "vsys_id", - "fieldValues": [ - 1 - ] - } - ], - "intervals": [ - "2020-08-15T00:00:00Z/2022-08-16T00:00:00Z" - ] - } - } - }, - "topSni": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "session_record", - "limit": "10000", - "parameters": { - "range": [ - { - "type": "eq", - "fieldKey": "vsys_id", - "fieldValues": [ - 1 - ] - } - ], - "intervals": [ - "2020-08-15T00:00:00.865+08:00/2022-08-16T00:00:00.865+08:00" - ] - } - } - }, - "subScriberidPool": { - "clientId":null, - "query":{ - "dataEngine":"AnalysisEngine", - "dataSource":"SUBSCRIBER_ID_VIEW", - "parameters":{ - "match":[ - { - "type":"exactly", - "fieldKey":"SUBSCRIBER_ID", - "fieldValues":[ - "test01", - "test02" - ] - } - ], - "range":[ - { - "type":"eq", - "fieldKey":"vsys_id", - "fieldValues":[ - 1 - ] - } - ] - } - } - }, - "gtpc": { - "clientId":null, - "query":{ - "dataEngine":"AnalysisEngine", - "dataSource":"gtpc_knowledge_base", - "parameters":{ - "match":[ - { - "type":"prefix", - "fieldKey":"phone_number", - "fieldValues":[ - "1761041" - ] - } - ], - "range":[ - { - "type":"eq", - "fieldKey":"vsys_id", - "fieldValues":[ - 1 - ] - } - ] - } - } - } -} diff --git a/src/test/resources/parameters/fieldDiscoveryTest.json b/src/test/resources/parameters/fieldDiscoveryTest.json new file mode 100644 index 00000000..e6ed275b --- /dev/null +++ b/src/test/resources/parameters/fieldDiscoveryTest.json @@ -0,0 +1,35 @@ +{ + "field_discovery_default": { + "name": "field_discovery", + "data_source": "session_record", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + }, + "field_discovery_sessions": { + "name": "field_discovery", + "data_source": "session_record", + "custom.field_discovery.metric": "sessions", + "custom.field_discovery.metric.fn": "count", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + }, + "field_discovery_bytes": { + "name": "field_discovery", + "data_source": "session_record", + "custom.field_discovery.metric": "bytes", + "custom.field_discovery.metric.fn": "sum", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + }, + "field_discovery_incoming_bytes": { + "name": "field_discovery", + "data_source": "session_record", + "custom.field_discovery.metric": "incoming_bytes", + "custom.field_discovery.metric.fn": "sum", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + }, + "field_discovery_outgoing_bytes": { + "name": "field_discovery", + "data_source": "session_record", + "custom.field_discovery.metric": "outgoing_bytes", + "custom.field_discovery.metric.fn": "sum", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/jobTest.json b/src/test/resources/parameters/jobTest.json deleted file mode 100644 index 777418ff..00000000 --- a/src/test/resources/parameters/jobTest.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "field_discovery_default": { - "query.type": "field_discovery", - "query.data_source": "session_record", - "custom.field_discovery.fields": [ - "log_id", - "security_action" - ], - "custom.field_discovery.filter": "vsys_id in (1,2) and client_ip='192.168.0.1' AND server_port = 80" - }, - "field_discovery_bytes": { - "query.type": "field_discovery", - "query.data_source": "session_record", - "custom.field_discovery.metric": "bytes", - "custom.field_discovery.metric.fn": "sum", - "custom.field_discovery.fields": [ - "security_action", - "proxy_action" - ], - "custom.field_discovery.filter": "vsys_id in (1,2) and client_ip='192.168.0.1' AND server_port = 80" - }, - "long_term": { - "query.type": "long_term", - "query.data_source": "session_record", - "custom.long_term.sql": "select client_ip, count(*) as count from session_record where vsys_id in (1,2) and client_ip='192.168.0.1' AND server_port = 80 group by client_ip order by count asc limit 10" - }, - "report": { - "query.type": "report", - "query.data_source": "session_record", - "custom.report.sql": "SELECT log_id, recv_time FROM session_record LIMIT 12 " - }, - "statistics_top": { - "query.type": "statistics", - "query.data_source": "session_record", - "custom.statistics.sql": "select client_ip, count(*) as count from session_record where vsys_id in (1,2) and client_ip='192.168.0.1' AND server_port = 80 group by client_ip order by count desc limit 10" - } -}
\ No newline at end of file diff --git a/src/test/resources/parameters/knowledgeBase.json b/src/test/resources/parameters/knowledgeBase.json index c5eadaaa..8c2bd42e 100644 --- a/src/test/resources/parameters/knowledgeBase.json +++ b/src/test/resources/parameters/knowledgeBase.json @@ -1,7 +1,21 @@ { - "publishTest": { + "publish": { + "kb_id": "test", "name": "test", - "format": "test", - "type": "test" + "format": "format", + "category": "category", + "is_valid": 1 + }, + "update": { + "kb_id": "test", + "version": "latest" + }, + "update_status": { + "kb_id": "test", + "version": "latest", + "is_valid": 0 + }, + "delete": { + "kb_id": "test" } }
\ No newline at end of file diff --git a/src/test/resources/parameters/recommendTest.json b/src/test/resources/parameters/recommendTest.json new file mode 100644 index 00000000..34ad5030 --- /dev/null +++ b/src/test/resources/parameters/recommendTest.json @@ -0,0 +1,20 @@ +{ + "ip_learning_fqdn_relate_ip": { + "name": "ip-learning-fqdn-relate-ip", + "filter": "VSYS_ID in (1,2,3,4,5) AND PROTOCOL in ('SSL', 'HTTP', 'DNS') AND DEPTH = 1 and UNIQ_NAME > 12 AND FQDN_NAME in ('google.com', 'itunes.apple.com')", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "limit": 100 + }, + "ip_learning_active_ip": { + "name": "ip-learning-active-ip", + "execution_mode": "oneshot", + "filter": "vsys_id in (1) AND 1=1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "order_by": "BYTES_TOTAL desc, LAST_FOUND_TIME desc", + "limit": 1 + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/sqlAdHocTest.json b/src/test/resources/parameters/sqlAdHocTest.json new file mode 100644 index 00000000..efc526fd --- /dev/null +++ b/src/test/resources/parameters/sqlAdHocTest.json @@ -0,0 +1,29 @@ +{ + "query_sql_default": { + "statement": "select * from session_record limit 1" + }, + "query_sql_oneshot": { + "statement": "select * from session_record limit 1", + "execution_mode": "oneshot" + }, + "query_sql_normal": { + "statement": "select * from session_record limit 1", + "execution_mode": "normal" + }, + "query_sql_blocking": { + "statement": "select * from session_record limit 1", + "execution_mode": "blocking" + }, + "query_sql_json": { + "statement": "select * from session_record limit 1", + "output_mode": "json" + }, + "query_sql_csv": { + "statement": "select * from session_record limit 1", + "output_mode": "csv" + }, + "query_sql_oneshot_error_trigger_sub_query": { + "statement": "SELECT COUNT_DISTINCT(client_ip) AS \"Client IP\" FROM security_event WHERE ((security_action = 'Deny')) AND recv_time >= UNIX_TIMESTAMP('2024-03-12T00:00:00+08:00') AND recv_time < UNIX_TIMESTAMP('2024-03-12T23:59:59+08:00') AND security_event.vsys_id IN (1) ORDER BY \"Client IP\" DESC LIMIT 20", + "execution_mode": "oneshot" + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/sqlSavedTest.json b/src/test/resources/parameters/sqlSavedTest.json new file mode 100644 index 00000000..dd128a67 --- /dev/null +++ b/src/test/resources/parameters/sqlSavedTest.json @@ -0,0 +1,6 @@ +{ + "default": { + "statement": "select * from session_record limit 1", + "is_saved_query": 1 + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/unstructuredTest.json b/src/test/resources/parameters/unstructuredTest.json deleted file mode 100644 index 64412dd7..00000000 --- a/src/test/resources/parameters/unstructuredTest.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "all": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "tsg_galaxy_v3", - "limit": "1000", - "parameters": { - "intervals": [ - "2023-03-01T00:00:00+08:00/2023-04-06T00:00:00+08:00" - ] - } - } - }, - "mail": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "session_record", - "limit": "1000", - "parameters": { - "intervals": [ - "2023-03-01T00:00:00+08:00/2023-04-06T00:00:00+08:00" - ] - } - } - }, - "http": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "security_event", - "limit": "1000", - "parameters": { - "intervals": [ - "2023-03-01T00:00:00+08:00/2023-04-06T00:00:00+08:00" - ] - } - } - }, - "pcap": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "voip_record", - "limit": "1000", - "parameters": { - "intervals": [ - "2023-03-01T00:00:00+08:00/2023-04-06T00:00:00+08:00" - ] - } - } - } -}
\ No newline at end of file |