Merge branch 'develop' into 'main'main

Develop

See merge request galaxy/platform/tsg-olap-troubleshooting!2

4 files changed, 325 insertions, 55 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..71a30e2
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,33 @@
+image: 192.168.40.153:8082/common/maven:3.8.1-openjdk-11-slim
+
+stages:
+  - build
+  - deploy
+
+
+variables:
+  VERSION: "$CI_COMMIT_TAG"
+
+build:
+  stage: build
+  script:
+    - echo "Building the project..."
+    - tar -czvf tsg-olap-troubleshooting-$VERSION.tar.gz bin/ config/ e2e-mockdata-generator/ README.md
+    - ls -lah
+  artifacts:
+    paths:
+      - tsg-olap-troubleshooting-$CI_COMMIT_TAG.tar.gz 
+  only:
+    - tags
+
+
+deploy:
+  stage: deploy
+  script:
+    - echo "Uploading to Nexus..."
+    - ls -lah
+    - curl -v -u $NEXUS_USER:$NEXUS_PASSWORD --upload-file tsg-olap-troubleshooting-$VERSION.tar.gz "$NEXUS_URL/troubleshooting/tsg-olap-troubleshooting-$VERSION.tar.gz"
+  dependencies:
+    - build 
+  only:
+    - tags
\ No newline at end of file
diff --git a/README.md b/README.md
index 9c0f756..4c6235b 100644
--- a/README.md
+++ b/README.md
@@ -1,22 +1,46 @@
 # TSG OLAP TROUBLESHOOTING
 
-
 ## End-to-End Test
 
 ### 概述
 
-TSG OLAP 支持对Logs，Metrics和Files进行端到端业务自检,自检数据流： Smaple Datasets -> Kafka Topic -> FLINK ETL -> Storage DB -> QGW API .
+支持对Logs，Metrics和Files进行端到端业务自检。
+数据流：`Smaple Datasets -> Kafka Topic -> FLINK ETL -> Storage DB -> QGW API`。
 
 ### 环境依赖
 
-- 样例数据生成工具：`e2e-mockdata-generator` ，需要JDK11环境
-- Report诊断工具：`Newman`，具体参考[帮助文档](https://learning.postman.com/docs/collections/using-newman-cli/newman-options/)
+- `e2e-mockdata-generator`样例数据生成工具，需JDK11环境
+- `Newman` 使用参考[帮助文档](https://learning.postman.com/docs/collections/using-newman-cli/newman-options/)
 
-### 使用方法
+### 命令详解
 
-#### 修改 enviroment.json 配置
+`./e2e_test.sh -h`
 
-`Newman CLI 使用QGW HTTP REST API 输出诊断报告,需要添加访问IP`
+```shell
+Usage: ./e2e_test.sh [options]
+
+Options:
+  -g <type>           Generate data (logs, metrics, files)
+  -d <type>           Run diagnostic report (logs, metrics, files)
+  -c                  Clear test data(Only support clear files)
+  -a                  Perform all operations: generate data, run diagnostics, and clear data
+  -i <key=value,...>  Set environment variable (data_center, hos_ip)
+  -v                  Enable verbose reporting
+  -e                  Enable emojitrain reporting
+  -h                  Show this help message
+
+Examples:
+  ./e2e_test.sh -a -e                                   Perform all operations and enable emojitrain reporting
+  ./e2e_test.sh -g logs -i data_center=my_data_center   Generate log data at my_data_center
+  ./e2e_test.sh -d logs -v                              Run diagnostics on logs data with verbose reporting
+  ./e2e_test.sh -d metrics -v                           Run diagnostics on metrics data with verbose reporting
+  ./e2e_test.sh -g logs                                 Generate log data use default data center
+  ./e2e_test.sh -c                                      Clear test data
+  
+```
+### 使用步骤
+#### Step1: 修改`enviroment.json`
+Newman CLI使用QGW HTTP REST API 输出诊断报告,需要添加访问IP。
 ```json
 [
   {
@@ -33,7 +57,7 @@ TSG OLAP 支持对Logs，Metrics和Files进行端到端业务自检,自检数据
   }
 ]
 ```
-#### 修改e2e_test.sh配置
+#### Step2: 修改`e2e_test.sh`
 
 - 增加每个分中心的kafka broker地址
 
@@ -53,34 +77,35 @@ TSG OLAP 支持对Logs，Metrics和Files进行端到端业务自检,自检数据
   )
 ```
 
-#### 命令详解
 
-`./e2e_test.sh -h`
 
-```shell
-Usage: ./e2e_test.sh [options]
+## Function Test
+### 概述
+支持对QGW API进行功能验证，包括：查询、数据集、数据库、工具、故障诊断、HOS、管理等。
 
-Options:
-  -g <type>           Generate data (logs, metrics, files)
-  -d <type>           Run diagnostic report (logs, metrics, files)
-  -c                  Clear test data(Only support clear files)
-  -a                  Perform all operations: generate data, run diagnostics, and clear data
-  -i <key=value,...>  Set environment variable (data_center, hos_ip)
-  -v                  Enable verbose reporting
-  -e                  Enable emojitrain reporting
-  -h                  Show this help message
+### 环境依赖
+- `Newman` 使用参考[帮助文档](https://learning.postman.com/docs/collections/using-newman-cli/newman-options/)
 
-Examples:
-  ./e2e_test.sh -a -e                                   Perform all operations and enable emojitrain reporting
-  ./e2e_test.sh -g logs -i data_center=my_data_center   Generate log data at my_data_center
-  ./e2e_test.sh -d logs -v                              Run diagnostics on logs data with verbose reporting
-  ./e2e_test.sh -d metrics -v                           Run diagnostics on metrics data with verbose reporting
-  ./e2e_test.sh -g logs                                 Generate log data use default data center
-  ./e2e_test.sh -c                                      Clear test data
-  
+### 命令详解
 
-```
+`./function_test.sh -h`
 
-## Function Test
+```shell
+Usage: function_test.sh [-f FOLDER_NUMBER] [-a] [-v] [-e]
+
+Options:
+  -f FOLDER_NUMBER  Specify a single folder to run from a collection:
+                    1. Query
+                    2. Dataset
+                    3. Database
+                    4. Util
+                    5. Troubleshooting
+                    6. HOS
+                    7. Knowledge Base File
+  -a                Run all options [1-7]
+  -v                Enable verbose reporting
+  -e                Enable emojitrain reporting
+  -h, --help        Show this help message and exit
+```
 
 
diff --git a/bin/e2e_test.sh b/bin/e2e_test.sh
index 11e6ec9..eb13777 100755
--- a/bin/e2e_test.sh
+++ b/bin/e2e_test.sh
@@ -85,15 +85,23 @@ generate_files() {
   cd $E2E_MOCKDATA_GENERATOR_PATH || { echo "Error: Cannot change directory to $E2E_MOCKDATA_GENERATOR_PATH"; exit 1; }
   # 123e4567-e89b-12d3-a456-426614174006 troubleshooting_pcapng
   # 123e4567-e89b-12d3-a456-426614174007 datapath_telemetry_pcapng
-  declare -A FILES=(
+  declare -A FILE_TYPES=(
     ["123e4567-e89b-12d3-a456-426614174001"]="traffic_pcapng"
     ["123e4567-e89b-12d3-a456-426614174002"]="html"
     ["123e4567-e89b-12d3-a456-426614174003"]="html"
     ["123e4567-e89b-12d3-a456-426614174004"]="eml"
     ["123e4567-e89b-12d3-a456-426614174005"]="traffic_pcapng"
   )
-    for uuid in "${!FILES[@]}"; do
-      java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --bootstrap_server $bootstrap_server --topic TRAFFIC-FILE-STREAM-RECORD -n $uuid --file_type ${FILES[$uuid]}
+ declare -A FILE_TOPICS=(
+    ["123e4567-e89b-12d3-a456-426614174001"]="TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD"
+    ["123e4567-e89b-12d3-a456-426614174002"]="TRAFFIC-HTTP-FILE-STREAM-RECORD"
+    ["123e4567-e89b-12d3-a456-426614174003"]="TRAFFIC-HTTP-FILE-STREAM-RECORD"
+    ["123e4567-e89b-12d3-a456-426614174004"]="TRAFFIC-EML-FILE-STREAM-RECORD"
+    ["123e4567-e89b-12d3-a456-426614174005"]="TRAFFIC-RTP-FILE-STREAM-RECORD"
+  )
+
+    for uuid in "${!FILE_TYPES[@]}"; do
+      java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --bootstrap_server $bootstrap_server --topic ${FILE_TOPICS[$uuid]} -n $uuid --file_type ${FILE_TYPES[$uuid]}
     done
 }
 
diff --git a/config/tsg-olap-e2e-test-collection.json b/config/tsg-olap-e2e-test-collection.json
index 0253c90..75f425f 100644
--- a/config/tsg-olap-e2e-test-collection.json
+++ b/config/tsg-olap-e2e-test-collection.json
@@ -1,7 +1,7 @@
 {
 	"info": {
-		"_postman_id": "8d632c51-8d7b-426a-ae7d-baaf5d41dca3",
-		"name": "tsg-olap-e2e-test-24.08",
+		"_postman_id": "15fc359a-af82-41a5-9dcc-f931fa7e534c",
+		"name": "tsg-olap-e2e-test-24.09",
 		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
 		"_exporter_id": "8105037"
 	},
@@ -1001,7 +1001,7 @@
 							"urlencoded": []
 						},
 						"url": {
-							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_file_bucket/123e4567-e89b-12d3-a456-426614174001",
+							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_policy_capture_file_bucket/123e4567-e89b-12d3-a456-426614174001",
 							"protocol": "http",
 							"host": [
 								"{{hos_ip}}"
@@ -1009,7 +1009,7 @@
 							"port": "{{hos_port}}",
 							"path": [
 								"hos",
-								"traffic_file_bucket",
+								"traffic_policy_capture_file_bucket",
 								"123e4567-e89b-12d3-a456-426614174001"
 							]
 						}
@@ -1017,7 +1017,7 @@
 					"response": []
 				},
 				{
-					"name": "http_request_body_test",
+					"name": "http_request_file_test",
 					"event": [
 						{
 							"listen": "test",
@@ -1062,7 +1062,7 @@
 							"urlencoded": []
 						},
 						"url": {
-							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_file_bucket/123e4567-e89b-12d3-a456-426614174002",
+							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_http_file_bucket/123e4567-e89b-12d3-a456-426614174002",
 							"protocol": "http",
 							"host": [
 								"{{hos_ip}}"
@@ -1070,7 +1070,7 @@
 							"port": "{{hos_port}}",
 							"path": [
 								"hos",
-								"traffic_file_bucket",
+								"traffic_http_file_bucket",
 								"123e4567-e89b-12d3-a456-426614174002"
 							]
 						}
@@ -1078,7 +1078,7 @@
 					"response": []
 				},
 				{
-					"name": "http_response_body_test",
+					"name": "http_response_file_test",
 					"event": [
 						{
 							"listen": "test",
@@ -1123,7 +1123,7 @@
 							"urlencoded": []
 						},
 						"url": {
-							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_file_bucket/123e4567-e89b-12d3-a456-426614174003",
+							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_http_file_bucket/123e4567-e89b-12d3-a456-426614174003",
 							"protocol": "http",
 							"host": [
 								"{{hos_ip}}"
@@ -1131,7 +1131,7 @@
 							"port": "{{hos_port}}",
 							"path": [
 								"hos",
-								"traffic_file_bucket",
+								"traffic_http_file_bucket",
 								"123e4567-e89b-12d3-a456-426614174003"
 							]
 						}
@@ -1184,7 +1184,7 @@
 							"urlencoded": []
 						},
 						"url": {
-							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_file_bucket/123e4567-e89b-12d3-a456-426614174004",
+							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_eml_file_bucket/123e4567-e89b-12d3-a456-426614174004",
 							"protocol": "http",
 							"host": [
 								"{{hos_ip}}"
@@ -1192,7 +1192,7 @@
 							"port": "{{hos_port}}",
 							"path": [
 								"hos",
-								"traffic_file_bucket",
+								"traffic_eml_file_bucket",
 								"123e4567-e89b-12d3-a456-426614174004"
 							]
 						}
@@ -1200,7 +1200,7 @@
 					"response": []
 				},
 				{
-					"name": "rtp_pcap_path_test",
+					"name": "rtp_pcap_file_test",
 					"event": [
 						{
 							"listen": "test",
@@ -1245,7 +1245,7 @@
 							"urlencoded": []
 						},
 						"url": {
-							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_file_bucket/123e4567-e89b-12d3-a456-426614174005",
+							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_rtp_file_bucket/123e4567-e89b-12d3-a456-426614174005",
 							"protocol": "http",
 							"host": [
 								"{{hos_ip}}"
@@ -1253,7 +1253,7 @@
 							"port": "{{hos_port}}",
 							"path": [
 								"hos",
-								"traffic_file_bucket",
+								"traffic_rtp_file_bucket",
 								"123e4567-e89b-12d3-a456-426614174005"
 							]
 						}
@@ -1398,7 +1398,7 @@
 			"name": "clear_test_data",
 			"item": [
 				{
-					"name": "delete_traffic_files_test",
+					"name": "delete_monitor_packet_capture_file",
 					"event": [
 						{
 							"listen": "test",
@@ -1437,7 +1437,7 @@
 						],
 						"body": {
 							"mode": "raw",
-							"raw": "<Delete>\n    <Quiet>true</Quiet>\n    <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174001</Key>\n    </Object>\n    <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174002</Key>\n    </Object>\n    <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174003</Key>\n    </Object>\n    <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174004</Key>\n    </Object>\n    <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174005</Key>\n    </Object>\n</Delete>",
+							"raw": "<Delete>\n    <Quiet>true</Quiet>\n    <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174001</Key>\n    </Object>\n</Delete>",
 							"options": {
 								"raw": {
 									"language": "xml"
@@ -1445,7 +1445,7 @@
 							}
 						},
 						"url": {
-							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_file_bucket?delete",
+							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_policy_capture_file_bucket?delete",
 							"protocol": "http",
 							"host": [
 								"{{hos_ip}}"
@@ -1453,7 +1453,211 @@
 							"port": "{{hos_port}}",
 							"path": [
 								"hos",
-								"traffic_file_bucket"
+								"traffic_policy_capture_file_bucket"
+							],
+							"query": [
+								{
+									"key": "delete",
+									"value": null
+								}
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "delete_http_file",
+					"event": [
+						{
+							"listen": "test",
+							"script": {
+								"exec": [
+									"pm.test(\"Status code is 200\", function () {",
+									"   pm.response.to.not.be.error;",
+									"});",
+									"",
+									"",
+									""
+								],
+								"type": "text/javascript",
+								"packages": {}
+							}
+						},
+						{
+							"listen": "prerequest",
+							"script": {
+								"exec": [
+									""
+								],
+								"type": "text/javascript",
+								"packages": {}
+							}
+						}
+					],
+					"request": {
+						"method": "POST",
+						"header": [
+							{
+								"key": "token",
+								"value": "{{hos_token}}",
+								"type": "text"
+							}
+						],
+						"body": {
+							"mode": "raw",
+							"raw": "<Delete>\n    <Quiet>true</Quiet>\n     <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174002</Key>\n    </Object>\n    <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174003</Key>\n    </Object>\n</Delete>",
+							"options": {
+								"raw": {
+									"language": "xml"
+								}
+							}
+						},
+						"url": {
+							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_http_file_bucket?delete",
+							"protocol": "http",
+							"host": [
+								"{{hos_ip}}"
+							],
+							"port": "{{hos_port}}",
+							"path": [
+								"hos",
+								"traffic_http_file_bucket"
+							],
+							"query": [
+								{
+									"key": "delete",
+									"value": null
+								}
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "delete_mail_eml_file",
+					"event": [
+						{
+							"listen": "test",
+							"script": {
+								"exec": [
+									"pm.test(\"Status code is 200\", function () {",
+									"   pm.response.to.not.be.error;",
+									"});",
+									"",
+									"",
+									""
+								],
+								"type": "text/javascript",
+								"packages": {}
+							}
+						},
+						{
+							"listen": "prerequest",
+							"script": {
+								"exec": [
+									""
+								],
+								"type": "text/javascript",
+								"packages": {}
+							}
+						}
+					],
+					"request": {
+						"method": "POST",
+						"header": [
+							{
+								"key": "token",
+								"value": "{{hos_token}}",
+								"type": "text"
+							}
+						],
+						"body": {
+							"mode": "raw",
+							"raw": "<Delete>\n    <Quiet>true</Quiet>\n    <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174004</Key>\n    </Object>\n</Delete>",
+							"options": {
+								"raw": {
+									"language": "xml"
+								}
+							}
+						},
+						"url": {
+							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_eml_file_bucket?delete",
+							"protocol": "http",
+							"host": [
+								"{{hos_ip}}"
+							],
+							"port": "{{hos_port}}",
+							"path": [
+								"hos",
+								"traffic_eml_file_bucket"
+							],
+							"query": [
+								{
+									"key": "delete",
+									"value": null
+								}
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "delete_rtp_pcap_file",
+					"event": [
+						{
+							"listen": "test",
+							"script": {
+								"exec": [
+									"pm.test(\"Status code is 200\", function () {",
+									"   pm.response.to.not.be.error;",
+									"});",
+									"",
+									"",
+									""
+								],
+								"type": "text/javascript",
+								"packages": {}
+							}
+						},
+						{
+							"listen": "prerequest",
+							"script": {
+								"exec": [
+									""
+								],
+								"type": "text/javascript",
+								"packages": {}
+							}
+						}
+					],
+					"request": {
+						"method": "POST",
+						"header": [
+							{
+								"key": "token",
+								"value": "{{hos_token}}",
+								"type": "text"
+							}
+						],
+						"body": {
+							"mode": "raw",
+							"raw": "<Delete>\n    <Quiet>true</Quiet>\n    <Object>\n        <Key>123e4567-e89b-12d3-a456-426614174005</Key>\n    </Object>\n</Delete>",
+							"options": {
+								"raw": {
+									"language": "xml"
+								}
+							}
+						},
+						"url": {
+							"raw": "http://{{hos_ip}}:{{hos_port}}/hos/traffic_rtp_file_bucket?delete",
+							"protocol": "http",
+							"host": [
+								"{{hos_ip}}"
+							],
+							"port": "{{hos_port}}",
+							"path": [
+								"hos",
+								"traffic_rtp_file_bucket"
 							],
 							"query": [
 								{
@@ -1509,7 +1713,7 @@
 							"urlencoded": []
 						},
 						"url": {
-							"raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/hos/traffic_file_bucket?file_names=123e4567-e89b-12d3-a456-426614174007",
+							"raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/hos/troubleshooting_file_bucket?file_names=123e4567-e89b-12d3-a456-426614174007",
 							"protocol": "http",
 							"host": [
 								"{{qgw_ip}}"
@@ -1518,7 +1722,7 @@
 							"path": [
 								"v1",
 								"hos",
-								"traffic_file_bucket"
+								"troubleshooting_file_bucket"
 							],
 							"query": [
 								{