Metric datasets use current unix timestamp and add traffic skecth metric dataset

4 files changed, 95 insertions, 26 deletions

diff --git a/24.06/bin/e2e_test.sh b/24.06/bin/e2e_test.sh
index 0d47f15..ddb3d64 100755
--- a/24.06/bin/e2e_test.sh
+++ b/24.06/bin/e2e_test.sh
@@ -12,17 +12,22 @@ E2E_MOCKDATA_GENERATOR_PATH="$BASE_DIR/e2e-mockdata-generator"
 CONFIG_PATH="$BASE_DIR/config"
 COLLECTION="tsg-olap-e2e-test-collection.json"
 ENVIRONMENT="$BASE_DIR/environment.json"
-DATA_CENTER="tsg_olap_dc_a" # Default Data Center 
+DATA_CENTER="tsg_olap" # Default Data Center 
 OTHER_VAR=""
+current=`date "+%Y-%m-%d %H:%M:%S"`
+timeStamp=`date -d "$current" +%s`
+currentTimeStamp=$((timeStamp*1000+10#`date "+%N"`/1000000))
 
 # data_center_name:kafka_server
-declare -A KAFKA_SERVERS=(
+declare -A  KAFKA_SERVERS=(
+  ["tsg_olap"]="192.168.44.11:9092"
   ["tsg_olap_dc_a"]="192.168.44.11:9092"
   ["tsg_olap_dc_b"]="192.168.44.11:9092"
 )
 
 # data_center_name:hos_endpoint
-declare -A HOS_ENDPOINTS=(
+HOS_ENDPOINTS=(
+  ["tsg_olap"]="192.168.44.11"
   ["tsg_olap_dc_a"]="192.168.44.11"
   ["tsg_olap_dc_b"]="192.168.44.11"
 )
@@ -58,10 +63,14 @@ generate_metrics() {
     ["POLICY-RULE-METRIC"]="policy_rule_metric.dat"
     ["OBJECT-STATISTICS-METRIC"]="object_statistics_metric.dat"
     ["STATISTICS-RULE-METRIC"]="statistics_rule_metric.dat"
+    ["TRAFFIC-SKETCH-METRIC"]="traffic_sketch_metric.dat"
   )
     for topic in "${!METRICS[@]}"; do
       # Replace the "data_center" value in the file
       sed -i "s/\"data_center\":\"[^\"]*\"/\"data_center\":\"$data_center\"/g" ./datasets/metrics/${METRICS[$topic]}
+      # Replace the "timestamp_ms" value in the file
+      sed -i "s/\"timestamp_ms\":[0-9]*/\"timestamp_ms\":$currentTimeStamp/" ./datasets/metrics/${METRICS[$topic]}
+
       java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --bootstrap_server $bootstrap_server --topic $topic -f ./datasets/metrics/${METRICS[$topic]}
     done
 }
@@ -120,7 +129,6 @@ run_diagnostic() {
    newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 -e $CONFIG_PATH/environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure $verbose --ignore-redirects $env_var_str --folder $folder $emojitrain
 }
 
-
 clear_data() {
   log_message "Clearing test data..."
   newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 --delay-request 500 -e $CONFIG_PATH/environment.json --ignore-redirects --folder clear_test_data -r emojitrain
diff --git a/24.06/config/tsg-olap-e2e-test-collection.json b/24.06/config/tsg-olap-e2e-test-collection.json
index 4255ec2..89c27f6 100644
--- a/24.06/config/tsg-olap-e2e-test-collection.json
+++ b/24.06/config/tsg-olap-e2e-test-collection.json
@@ -1,7 +1,7 @@
 {
 	"info": {
 		"_postman_id": "8d632c51-8d7b-426a-ae7d-baaf5d41dca3",
-		"name": "tsg-olap-e2e-test-24.05",
+		"name": "tsg-olap-e2e-test-24.06",
 		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
 		"_exporter_id": "8105037"
 	},
@@ -10,7 +10,7 @@
 			"name": "logs",
 			"item": [
 				{
-					"name": "proxy_event_test",
+					"name": "session_record_test",
 					"event": [
 						{
 							"listen": "test",
@@ -20,7 +20,7 @@
 									"   pm.response.to.not.be.error;",
 									"   pm.response.to.be.json;",
 									"   const responseJson = pm.response.json();",
-									"   pm.expect(Number(responseJson.data[0].cnt)).to.at.least(1);",
+									"   pm.expect(Number(responseJson.data[0].cnt)).to.at.least(7);",
 									"});",
 									"",
 									"",
@@ -50,7 +50,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from proxy_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from session_record where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\",\n   \"output_mode\":\"json\",\n   \"execution_mode\":\"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -74,7 +74,7 @@
 					"response": []
 				},
 				{
-					"name": "voip_record_test",
+					"name": "proxy_event_test",
 					"event": [
 						{
 							"listen": "test",
@@ -114,7 +114,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from voip_record where vsys_id=1024 and decoded_as='VoIP' and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from proxy_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -138,7 +138,7 @@
 					"response": []
 				},
 				{
-					"name": "monitor_event_test",
+					"name": "voip_record_test",
 					"event": [
 						{
 							"listen": "test",
@@ -148,7 +148,7 @@
 									"   pm.response.to.not.be.error;",
 									"   pm.response.to.be.json;",
 									"   const responseJson = pm.response.json();",
-									"   pm.expect(Number(responseJson.data[0].cnt)).to.at.least(7);",
+									"   pm.expect(Number(responseJson.data[0].cnt)).to.at.least(1);",
 									"});",
 									"",
 									"",
@@ -178,7 +178,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from monitor_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\":\"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from voip_record where vsys_id=1024 and decoded_as='VoIP' and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -202,7 +202,7 @@
 					"response": []
 				},
 				{
-					"name": "security_event_test",
+					"name": "monitor_event_test",
 					"event": [
 						{
 							"listen": "test",
@@ -242,7 +242,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from security_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\":\"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from monitor_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\":\"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -266,7 +266,7 @@
 					"response": []
 				},
 				{
-					"name": "session_record_test",
+					"name": "security_event_test",
 					"event": [
 						{
 							"listen": "test",
@@ -306,7 +306,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from session_record where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\",\n   \"output_mode\":\"json\",\n   \"execution_mode\":\"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from security_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\":\"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -375,7 +375,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from traffic_general_stat where __time = {{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from traffic_general_stat where __time >={{Last 10 Minutes Start}}  and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -439,7 +439,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from application_protocol_stat where __time = {{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from application_protocol_stat where __time >={{Last 10 Minutes Start}}  and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -503,7 +503,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center,count(*) as cnt from statistics_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center,count(*) as cnt from statistics_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -567,7 +567,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from security_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from security_rule_hits where __time >={{Last 10 Minutes Start}}  and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -627,7 +627,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from monitor_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from monitor_rule_hits where __time>={{Last 10 Minutes Start}}  and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -687,7 +687,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center,count(*) as cnt from proxy_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center,count(*) as cnt from proxy_rule_hits where __time >={{Last 10 Minutes Start}}  and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -747,7 +747,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from service_chaining_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from service_chaining_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -807,7 +807,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from object_statistics where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from object_statistics where __time >={{Last 10 Minutes Start}}  and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -867,7 +867,67 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from statistics_rule where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from statistics_rule where __time>={{Last 10 Minutes Start}}  and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/query/sql",
+							"protocol": "http",
+							"host": [
+								"{{qgw_ip}}"
+							],
+							"port": "{{qgw_port}}",
+							"path": [
+								"v1",
+								"query",
+								"sql"
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "traffic_sketch_metric_test",
+					"event": [
+						{
+							"listen": "test",
+							"script": {
+								"exec": [
+									"pm.test(\"Status code is 200\", function () {",
+									"   pm.response.to.not.be.error;",
+									"   pm.response.to.be.json;",
+									"   const responseJson = pm.response.json();",
+									"   pm.expect(Number(responseJson.data[0].cnt)).to.at.least(1);",
+									"});",
+									"",
+									"",
+									""
+								],
+								"type": "text/javascript",
+								"packages": {}
+							}
+						},
+						{
+							"listen": "prerequest",
+							"script": {
+								"exec": [
+									""
+								],
+								"type": "text/javascript",
+								"packages": {}
+							}
+						}
+					],
+					"request": {
+						"method": "POST",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n   \"statement\" : \"select data_center, count(*) as cnt from traffic_sketch_metric where recv_time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n   \"output_mode\":\"json\",\n   \"execution_mode\": \"oneshot\"\n}",
 							"options": {
 								"raw": {
 									"language": "json"
diff --git a/24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat b/24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat
new file mode 100644
index 0000000..4f80dbc
--- /dev/null
+++ b/24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat
@@ -0,0 +1 @@
+{"name":"traffic_sketch_metric","tags":{"direction":"Outbound","c2s_link_id":0,"s2c_link_id":0,"ip_protocol":"","client_ip":"0.0.0.0","server_ip":"0.0.0.0","server_country":"","client_asn":0,"server_asn":0,"server_fqdn":"","server_domain":"","app":"","app_category":"","c2s_ttl":0,"s2c_ttl":0,"device_id":"0000000000000000","device_group":"tsg_olap","data_center":"tsg_olap","vsys_id":1024},"fields":{"sessions":0,"bytes":0,"sent_bytes":0,"received_bytes":0,"pkts":0,"sent_pkts":0,"received_pkts":0},"timestamp_ms":946681200000}
diff --git a/troubleshooting.tar.gz b/troubleshooting.tar.gz
new file mode 100644
index 0000000..d01a75d
--- /dev/null
+++ b/troubleshooting.tar.gz