diff options
| -rwxr-xr-x | 24.06/bin/e2e_test.sh | 16 | ||||
| -rw-r--r-- | 24.06/config/tsg-olap-e2e-test-collection.json | 104 | ||||
| -rw-r--r-- | 24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat | 1 | ||||
| -rw-r--r-- | troubleshooting.tar.gz | bin | 0 -> 18809841 bytes |
4 files changed, 95 insertions, 26 deletions
diff --git a/24.06/bin/e2e_test.sh b/24.06/bin/e2e_test.sh index 0d47f15..ddb3d64 100755 --- a/24.06/bin/e2e_test.sh +++ b/24.06/bin/e2e_test.sh @@ -12,17 +12,22 @@ E2E_MOCKDATA_GENERATOR_PATH="$BASE_DIR/e2e-mockdata-generator" CONFIG_PATH="$BASE_DIR/config" COLLECTION="tsg-olap-e2e-test-collection.json" ENVIRONMENT="$BASE_DIR/environment.json" -DATA_CENTER="tsg_olap_dc_a" # Default Data Center +DATA_CENTER="tsg_olap" # Default Data Center OTHER_VAR="" +current=`date "+%Y-%m-%d %H:%M:%S"` +timeStamp=`date -d "$current" +%s` +currentTimeStamp=$((timeStamp*1000+10#`date "+%N"`/1000000)) # data_center_name:kafka_server -declare -A KAFKA_SERVERS=( +declare -A KAFKA_SERVERS=( + ["tsg_olap"]="192.168.44.11:9092" ["tsg_olap_dc_a"]="192.168.44.11:9092" ["tsg_olap_dc_b"]="192.168.44.11:9092" ) # data_center_name:hos_endpoint -declare -A HOS_ENDPOINTS=( +HOS_ENDPOINTS=( + ["tsg_olap"]="192.168.44.11" ["tsg_olap_dc_a"]="192.168.44.11" ["tsg_olap_dc_b"]="192.168.44.11" ) @@ -58,10 +63,14 @@ generate_metrics() { ["POLICY-RULE-METRIC"]="policy_rule_metric.dat" ["OBJECT-STATISTICS-METRIC"]="object_statistics_metric.dat" ["STATISTICS-RULE-METRIC"]="statistics_rule_metric.dat" + ["TRAFFIC-SKETCH-METRIC"]="traffic_sketch_metric.dat" ) for topic in "${!METRICS[@]}"; do # Replace the "data_center" value in the file sed -i "s/\"data_center\":\"[^\"]*\"/\"data_center\":\"$data_center\"/g" ./datasets/metrics/${METRICS[$topic]} + # Replace the "timestamp_ms" value in the file + sed -i "s/\"timestamp_ms\":[0-9]*/\"timestamp_ms\":$currentTimeStamp/" ./datasets/metrics/${METRICS[$topic]} + java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --bootstrap_server $bootstrap_server --topic $topic -f ./datasets/metrics/${METRICS[$topic]} done } @@ -120,7 +129,6 @@ run_diagnostic() { newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 -e $CONFIG_PATH/environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure $verbose --ignore-redirects $env_var_str --folder $folder $emojitrain } - clear_data() { log_message "Clearing test data..." newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 --delay-request 500 -e $CONFIG_PATH/environment.json --ignore-redirects --folder clear_test_data -r emojitrain diff --git a/24.06/config/tsg-olap-e2e-test-collection.json b/24.06/config/tsg-olap-e2e-test-collection.json index 4255ec2..89c27f6 100644 --- a/24.06/config/tsg-olap-e2e-test-collection.json +++ b/24.06/config/tsg-olap-e2e-test-collection.json @@ -1,7 +1,7 @@ { "info": { "_postman_id": "8d632c51-8d7b-426a-ae7d-baaf5d41dca3", - "name": "tsg-olap-e2e-test-24.05", + "name": "tsg-olap-e2e-test-24.06", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "8105037" }, @@ -10,7 +10,7 @@ "name": "logs", "item": [ { - "name": "proxy_event_test", + "name": "session_record_test", "event": [ { "listen": "test", @@ -20,7 +20,7 @@ " pm.response.to.not.be.error;", " pm.response.to.be.json;", " const responseJson = pm.response.json();", - " pm.expect(Number(responseJson.data[0].cnt)).to.at.least(1);", + " pm.expect(Number(responseJson.data[0].cnt)).to.at.least(7);", "});", "", "", @@ -50,7 +50,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from proxy_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from session_record where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\",\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -74,7 +74,7 @@ "response": [] }, { - "name": "voip_record_test", + "name": "proxy_event_test", "event": [ { "listen": "test", @@ -114,7 +114,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from voip_record where vsys_id=1024 and decoded_as='VoIP' and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from proxy_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -138,7 +138,7 @@ "response": [] }, { - "name": "monitor_event_test", + "name": "voip_record_test", "event": [ { "listen": "test", @@ -148,7 +148,7 @@ " pm.response.to.not.be.error;", " pm.response.to.be.json;", " const responseJson = pm.response.json();", - " pm.expect(Number(responseJson.data[0].cnt)).to.at.least(7);", + " pm.expect(Number(responseJson.data[0].cnt)).to.at.least(1);", "});", "", "", @@ -178,7 +178,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from monitor_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from voip_record where vsys_id=1024 and decoded_as='VoIP' and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -202,7 +202,7 @@ "response": [] }, { - "name": "security_event_test", + "name": "monitor_event_test", "event": [ { "listen": "test", @@ -242,7 +242,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from security_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from monitor_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -266,7 +266,7 @@ "response": [] }, { - "name": "session_record_test", + "name": "security_event_test", "event": [ { "listen": "test", @@ -306,7 +306,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from session_record where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\",\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from security_event where vsys_id=1024 and recv_time >={{Last 10 Minutes Start}} and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\":\"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -375,7 +375,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from traffic_general_stat where __time = {{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from traffic_general_stat where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -439,7 +439,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from application_protocol_stat where __time = {{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from application_protocol_stat where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -503,7 +503,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from statistics_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from statistics_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -567,7 +567,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from security_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from security_rule_hits where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -627,7 +627,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from monitor_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from monitor_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -687,7 +687,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from proxy_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center,count(*) as cnt from proxy_rule_hits where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -747,7 +747,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from service_chaining_rule_hits where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from service_chaining_rule_hits where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -807,7 +807,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from object_statistics where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from object_statistics where __time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" @@ -867,7 +867,67 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from statistics_rule where __time ={{metric_fixed_time}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from statistics_rule where __time>={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://{{qgw_ip}}:{{qgw_port}}/v1/query/sql", + "protocol": "http", + "host": [ + "{{qgw_ip}}" + ], + "port": "{{qgw_port}}", + "path": [ + "v1", + "query", + "sql" + ] + } + }, + "response": [] + }, + { + "name": "traffic_sketch_metric_test", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.not.be.error;", + " pm.response.to.be.json;", + " const responseJson = pm.response.json();", + " pm.expect(Number(responseJson.data[0].cnt)).to.at.least(1);", + "});", + "", + "", + "" + ], + "type": "text/javascript", + "packages": {} + } + }, + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"statement\" : \"select data_center, count(*) as cnt from traffic_sketch_metric where recv_time >={{Last 10 Minutes Start}} and vsys_id=1024 and data_center='{{data_center}}' group by data_center order by cnt asc\" ,\n \"output_mode\":\"json\",\n \"execution_mode\": \"oneshot\"\n}", "options": { "raw": { "language": "json" diff --git a/24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat b/24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat new file mode 100644 index 0000000..4f80dbc --- /dev/null +++ b/24.06/e2e-mockdata-generator/datasets/metrics/traffic_sketch_metric.dat @@ -0,0 +1 @@ +{"name":"traffic_sketch_metric","tags":{"direction":"Outbound","c2s_link_id":0,"s2c_link_id":0,"ip_protocol":"","client_ip":"0.0.0.0","server_ip":"0.0.0.0","server_country":"","client_asn":0,"server_asn":0,"server_fqdn":"","server_domain":"","app":"","app_category":"","c2s_ttl":0,"s2c_ttl":0,"device_id":"0000000000000000","device_group":"tsg_olap","data_center":"tsg_olap","vsys_id":1024},"fields":{"sessions":0,"bytes":0,"sent_bytes":0,"received_bytes":0,"pkts":0,"sent_pkts":0,"received_pkts":0},"timestamp_ms":946681200000} diff --git a/troubleshooting.tar.gz b/troubleshooting.tar.gz Binary files differnew file mode 100644 index 0000000..d01a75d --- /dev/null +++ b/troubleshooting.tar.gz |