Support new troubleshooting in 24.05

11 files changed, 471 insertions, 111 deletions

diff --git a/24.05/README.md b/24.05/README.md
index 3bee145..52425d8 100644
--- a/24.05/README.md
+++ b/24.05/README.md
@@ -2,119 +2,78 @@
 
 ## 概述
 
-TSG OLAP的数据摄入分为三种类型：Logs、Metrics 和 File Chunks。为确保能够正确处理并写入相应存储系统，需要支持端到端业务自检。
+TSG OLAP 支持对Logs，Metrics和Files进行端到端业务自检,自检数据流： Smaple Datasets -> Kafka Topic -> FLINK ETL -> Storage DB -> QGW API .
 
 ## 环境依赖
 
-- 测试数据生成工具 `e2e-mockdata-generator.jar`
-- 安装`Newman`，详细参考[帮助文档](https://learning.postman.com/docs/collections/using-newman-cli/newman-options/)
+- 样例数据生成工具：`e2e-mockdata-generator` ，需要JDK11环境
+- Report诊断工具：`Newman`，具体参考[帮助文档](https://learning.postman.com/docs/collections/using-newman-cli/newman-options/)
 
 ## 使用方法
 
-### Newman CLI enviroment.json 修改如下配置
-`Newman CLI 使用QGW HTTP Rest API 接口输出诊断报告，一般部署至国家中心`
+### 修改 enviroment.json 配置
+
+`Newman CLI 使用QGW HTTP REST API 输出诊断报告,需要添加访问IP`
 ```json
+[
   {
-			"key": "qgw_ip",
-			"value": "192.168.44.30",
-			"type": "default",
-			"enabled": true
-		},
-		{
-			"key": "qgw_port",
-			"value": "9999",
-			"type": "default",
-			"enabled": true
-		}
-
+    "key": "qgw_ip",
+    "value": "127.0.0.1",
+    "type": "default",
+    "enabled": true
+  },
+  {
+    "key": "hos_token",
+    "value": "c21f969b5f03d33d43e04f8f136e7682",
+    "type": "secret",
+    "enabled": true
+  }
+]
 ```
-### 测试数据生成工具修改Kafka地址
+### 修改e2e_test.sh配置
 
-`Kafka需为分中心地址`
-
-- 修改`config.properties`, 增加Kafka访问地址
-```props
-  kafka.server=192.168.41.29:9092 
-```
+- 增加每个分中心的kafka broker地址
 
-### 写入测试集至Kafka
-- Logs
 ```shell
-
-cd e2e-mockdata-generator/
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic SESSION-RECORD  -f ./datasets/logs/session_record.dat
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic VOIP-RECORD  -f ./datasets/logs/voip_record.dat
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic PROXY-EVENT  -f ./datasets/logs/proxy_event.dat
-
+  # [data_center_name]:kafka_server_list
+  declare -A KAFKA_SERVERS=(
+    ["tsg_olap_dc_a"]="192.168.44.11:9092"
+    ["tsg_olap_dc_b"]="192.168.44.11:9092"
+  )
 ```
-- Metrics
-
+- 增加每个分中心的HOS访问地址
 ```shell
-cd e2e-mockdata-generator/
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic NETWORK-TRAFFIC-METRIC  -f ./datasets/metrics/network_traffic_metric.dat
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic POLICY-RULE-METRIC  -f ./datasets/metrics/policy_rule_metric.dat
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic OBJECT-STATISTICS-METRIC  -f ./datasets/metrics/object_statistics_metric.dat
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --topic STATISTICS-RULE-METRIC  -f ./datasets/metrics/statistics_rule_metric.dat
-
+  # [data_center_name]:hos_endpoint_uri
+  declare -A HOS_ENDPOINTS=(
+    ["tsg_olap_dc_a"]="192.168.44.11"
+    ["tsg_olap_dc_b"]="192.168.44.11"
+  )
 ```
 
-- Files
-	- 123e4567-e89b-12d3-a456-426614174001  监测策略 PcapNG
-	- 123e4567-e89b-12d3-a456-426614174002  HTTP Request Body
-	- 123e4567-e89b-12d3-a456-426614174003  HTTP Response Body
-	- 123e4567-e89b-12d3-a456-426614174004  MAIL EML
-	- 123e4567-e89b-12d3-a456-426614174005  RTP PcapNG
-	- 123e4567-e89b-12d3-a456-426614174006  Troubleshooting PcapNG
-	- 123e4567-e89b-12d3-a456-426614174007  Datapath Telemetry PcapNG
-
-```shell
-cd e2e-mockdata-generator/
+### 命令详解
 
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174001 --file_type traffic_pcapng
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174002 --file_type html
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174003 --file_type html
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174004 --file_type eml
-
-java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TRAFFIC-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174005 --file_type traffic_pcapng
-
-# java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic TROUBLESHOOTING-FILE-STREAM-RECORD -n 123e4567-e89b-12d3-a456-426614174006 --file_type troubleshooting_pcapng
-
-# java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --topic DATAPATH-TELEMETRY-RECORD -n 123e4567-e89b-12d3-a456-426614174007 --file_type datapath_telemetry_pcapng
-
-```
-
-### 输出故障诊断报告（等待3-5分钟）
+`./e2e_test.sh -h`
 
 ```shell
-
-# -folder logs ：对日志进行故障诊断，输出诊断明细；指定data_center。
-# -folder metrics：对Metrics进行故障诊断，输出诊断明细；指定data_center。
-# -folder files：对文件进行故障诊断，输出诊断明细；指定分中心HOS访问地址。
-newman run ./tsg-olap-e2e-test-collection.json -n 1 -e ./environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure  --verbose --ignore-redirects --env-var "data_center=tsg_olap"  --folder logs 
-
-newman run ./tsg-olap-e2e-test-collection.json -n 1 -e ./environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure  --verbose --ignore-redirects --env-var "data_center=tsg_olap"  --folder metrics 
-
-newman run ./tsg-olap-e2e-test-collection.json -n 1 -e ./environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure  --verbose --ignore-redirects --env-var "hos_ip=127.0.0.1"  --folder files 
-
-# -folder logs：对日志进行故障诊断，通过表情形式输出测试结果
-# -folder files：对文件进行故障诊断，通过表情形式输出测试结果
-newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e ./environment.json --env-var "data_center=tsg_olap" --ignore-redirects --folder logs -r emojitrain
-newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e ./environment.json  --env-var "hos_ip=127.0.0.1"  --ignore-redirects --folder files -r emojitrain
-
-#清除测试数据（暂支持对文件的删除）
-newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e ./environment.json --ignore-redirects --folder clear_test_data -r emojitrain
-
+Usage: ./e2e_test.sh [options]
+
+Options:
+  -g <type>           Generate data (logs, metrics, files)
+  -d <type>           Run diagnostic report (logs, metrics, files)
+  -c                  Clear test data
+  -a                  Perform all operations: generate data, run diagnostics, and clear data
+  -i <key=value,...>  Set environment variable (data_center, hos_ip)
+  -v                  Enable verbose reporting
+  -e                  Enable emojitrain reporting
+  -h                  Show this help message
+
+Examples:
+  ./e2e_test.sh -a -e                                   Perform all operations and enable emojitrain reporting
+  ./e2e_test.sh -g logs -i data_center=my_data_center   Generate log data at my_data_center
+  ./e2e_test.sh -d logs -v                              Run diagnostics on logs data with verbose reporting
+  ./e2e_test.sh -d metrics -v                           Run diagnostics on metrics data with verbose reporting
+  ./e2e_test.sh -g logs                                 Generate log data use default data center
+  ./e2e_test.sh -c                                      Clear test data
+  
 
 ```
-
-
diff --git a/24.05/bin/e2e_test.sh b/24.05/bin/e2e_test.sh
new file mode 100755
index 0000000..fb0e8e1
--- /dev/null
+++ b/24.05/bin/e2e_test.sh
@@ -0,0 +1,251 @@
+#!/bin/bash
+#BASE_DIR=$(cd $(dirname $0) && pwd)
+BASE_DIR="$(dirname "$(pwd)")"
+# Check if required tools are installed
+if ! command -v java &> /dev/null; then
+  echo "Error: Java is not installed."
+  exit 1
+fi
+
+# Define common variables
+E2E_MOCKDATA_GENERATOR_PATH="$BASE_DIR/e2e-mockdata-generator"
+CONFIG_PATH="$BASE_DIR/config"
+COLLECTION="tsg-olap-e2e-test-collection.json"
+ENVIRONMENT="$BASE_DIR/environment.json"
+DATA_CENTER="tsg_olap_dc_a" # Default Data Center 
+OTHER_VAR=""
+
+# data_center_name:kafka_server
+declare -A KAFKA_SERVERS=(
+  ["tsg_olap_dc_a"]="192.168.44.11:9092"
+  ["tsg_olap_dc_b"]="192.168.44.11:9092"
+)
+
+# data_center_name:hos_endpoint
+declare -A HOS_ENDPOINTS=(
+  ["tsg_olap_dc_a"]="192.168.44.11"
+  ["tsg_olap_dc_b"]="192.168.44.11"
+)
+
+log_message() {
+  echo "$(date '+%Y-%m-%d %H:%M:%S') - $1"
+}
+
+generate_logs() {
+  log_message "Generating logs for $1 $2 ..."
+  local data_center=$1
+  local bootstrap_server=$2
+  cd $E2E_MOCKDATA_GENERATOR_PATH || { echo "Error: Cannot change directory to $E2E_MOCKDATA_GENERATOR_PATH"; exit 1; }
+  declare -A LOGS=(
+    ["SESSION-RECORD"]="session_record.dat"
+    ["VOIP-RECORD"]="voip_record.dat"
+    ["PROXY-EVENT"]="proxy_event.dat"
+  )
+    for topic in "${!LOGS[@]}"; do
+      # Replace the "data_center" value in the file
+      sed -i "s/\"data_center\":\"[^\"]*\"/\"data_center\":\"$data_center\"/g" ./datasets/logs/${LOGS[$topic]}
+      java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --bootstrap_server $bootstrap_server   --topic $topic -f ./datasets/logs/${LOGS[$topic]}
+    done
+}
+
+generate_metrics() {
+  log_message "Generating metrics for $1 $2..."
+  local data_center=$1
+  local bootstrap_server=$2
+  cd $E2E_MOCKDATA_GENERATOR_PATH || { echo "Error: Cannot change directory to $E2E_MOCKDATA_GENERATOR_PATH"; exit 1; }
+  declare -A METRICS=(
+    ["NETWORK-TRAFFIC-METRIC"]="network_traffic_metric.dat"
+    ["POLICY-RULE-METRIC"]="policy_rule_metric.dat"
+    ["OBJECT-STATISTICS-METRIC"]="object_statistics_metric.dat"
+    ["STATISTICS-RULE-METRIC"]="statistics_rule_metric.dat"
+  )
+    for topic in "${!METRICS[@]}"; do
+      # Replace the "data_center" value in the file
+      sed -i "s/\"data_center\":\"[^\"]*\"/\"data_center\":\"$data_center\"/g" ./datasets/metrics/${METRICS[$topic]}
+      java -cp e2e-mockdata-generator.jar com.geedgenetworks.LogGenerator --bootstrap_server $bootstrap_server --topic $topic -f ./datasets/metrics/${METRICS[$topic]}
+    done
+}
+
+
+generate_files() {
+  log_message "Generating files for $1 $2 ..."
+  local data_center=$1
+  local bootstrap_server=$2
+  cd $E2E_MOCKDATA_GENERATOR_PATH || { echo "Error: Cannot change directory to $E2E_MOCKDATA_GENERATOR_PATH"; exit 1; }
+  # 123e4567-e89b-12d3-a456-426614174006 troubleshooting_pcapng
+  # 123e4567-e89b-12d3-a456-426614174007 datapath_telemetry_pcapng
+  declare -A FILES=(
+    ["123e4567-e89b-12d3-a456-426614174001"]="traffic_pcapng"
+    ["123e4567-e89b-12d3-a456-426614174002"]="html"
+    ["123e4567-e89b-12d3-a456-426614174003"]="html"
+    ["123e4567-e89b-12d3-a456-426614174004"]="eml"
+    ["123e4567-e89b-12d3-a456-426614174005"]="traffic_pcapng"
+  )
+    for uuid in "${!FILES[@]}"; do
+      java -cp e2e-mockdata-generator.jar com.geedgenetworks.FileChunkGenerator --bootstrap_server $bootstrap_server --topic TRAFFIC-FILE-STREAM-RECORD -n $uuid --file_type ${FILES[$uuid]}
+    done
+}
+
+
+# Define function to run diagnostic report
+  # -folder logs ：对日志进行故障诊断，输出诊断明细；指定data_center。
+  # -folder metrics：对Metrics进行故障诊断，输出诊断明细；指定data_center。
+  # -folder files：对文件进行故障诊断，输出诊断明细；指定分中心HOS访问地址。
+  # -folder logs：对日志进行故障诊断，通过表情形式输出测试结果
+  # -folder files：对文件进行故障诊断，通过表情形式输出测试结果
+  #  newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e $ENVIRONMENT --env-var "data_center=$DATA_CENTER" --ignore-redirects --folder logs -r emojitrain
+  #  newman run ./tsg-olap-e2e-test-collection.json -n 1 --delay-request 500 -e $ENVIRONMENT --env-var "hos_ip=$HOS_IP" --ignore-redirects --folder files -r emojitrain
+run_diagnostic() {
+  log_message "Running diagnostic for $1..."
+  cd $BASE_DIR || { echo "Error: Cannot change directory to $BASE_DIR"; exit 1; }
+  local folder=$1
+  local env_var=$2
+  local verbose=""
+  local emojitrain=""
+  if $verbose_flag; then
+    verbose="--verbose"
+  fi
+
+  if $emojitrain_flag; then
+    emojitrain="-r emojitrain"
+  fi
+  
+   newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 -e $CONFIG_PATH/environment.json --delay-request 500 --timeout-script 10000 --timeout-request 300000 --timeout 3600000 --insecure $verbose --ignore-redirects --env-var $env_var --folder $folder $emojitrain
+}
+
+
+clear_data() {
+  log_message "Clearing test data..."
+  newman run $CONFIG_PATH/tsg-olap-e2e-test-collection.json -g $CONFIG_PATH/globals.json -n 1 --delay-request 500 -e $CONFIG_PATH/environment.json --ignore-redirects --folder clear_test_data -r emojitrain
+}
+
+# Help message
+show_help() {
+  echo ""
+  echo "Usage: $0 [options]"
+  echo ""
+  echo "Options:"
+  echo "  -g <type>       Generate data (logs, metrics, files)"
+  echo "  -d <type>       Run diagnostic report (logs, metrics, files)"
+  echo "  -c              Clear test data"
+  echo "  -a              Perform all operations: generate data, run diagnostics, and clear data"
+  echo "  -i <key=value,...>"
+  echo "                  Set environment variable (data_center, hos_ip)"
+  echo "  -v              Enable verbose reporting"
+  echo "  -e              Enable emojitrain reporting"
+  echo "  -h              Show this help message"
+  echo ""
+  echo "Examples:"
+  echo "  $0 -a                                      Perform all operations"
+  echo "  $0 -g logs                                 Generate log data"
+  echo "  $0 -d metrics -v                           Run diagnostics on metrics data with verbose reporting"
+  echo "  $0 -c                                      Clear test data"
+  echo "  $0 -g logs -i data_center=my_data_center   Generate log data in my_data_center"
+  echo ""
+}
+
+# Initialize flags
+generate_flag=false
+diagnostic_flag=false
+verbose_flag=false
+emojitrain_flag=false
+clear_flag=false
+all_flag=false
+generate_type=""
+diagnostic_type=""
+
+
+# Parse command-line arguments
+while getopts ":g:d:i:acveh" opt; do
+  case $opt in
+    g)
+      generate_type=$OPTARG
+      generate_flag=true
+      ;;
+    d)
+      diagnostic_type=$OPTARG
+      diagnostic_flag=true
+      ;;    
+    c)
+      clear_flag=true
+      ;;
+    a)
+      all_flag=true
+      ;;
+    i)
+      # Split the input based on whitespace, then further split by '=' to assign key-value pairs
+      IFS=',' read -ra vars <<< "$OPTARG"
+      for var in "${vars[@]}"; do
+        # Remove leading and trailing whitespaces
+        var=$(echo "$var" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
+        IFS='=' read -r key value <<< "$var"
+        case $key in
+          data_center) DATA_CENTER=$value ;;
+          other_var) OTHER_VAR=$value ;;
+          *) echo "Invalid environment variable: $key"; show_help; exit 1 ;;
+        esac
+      done
+      ;;
+    v)
+      verbose_flag=true
+      ;; 
+    e)
+      emojitrain_flag=true
+      ;;
+    h)
+      show_help
+      exit 0
+      ;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      show_help
+      exit 1
+      ;;
+  esac
+done
+
+
+if $generate_flag; then
+ case $generate_type in
+    logs) generate_logs "$DATA_CENTER" "${KAFKA_SERVERS[$DATA_CENTER]}" ;;
+    metrics) generate_metrics "$DATA_CENTER" "${KAFKA_SERVERS[$DATA_CENTER]}" ;;
+    files) generate_files "$DATA_CENTER" "${KAFKA_SERVERS[$DATA_CENTER]}" ;;
+    *) echo "Invalid generate type"; show_help; exit 1 ;;
+  esac
+fi
+
+if $diagnostic_flag; then
+  case $diagnostic_type in
+    logs) run_diagnostic "logs" "data_center=$DATA_CENTER" ;;
+    metrics) run_diagnostic "metrics" "data_center=$DATA_CENTER" ;;
+    files) run_diagnostic "files" "data_center=$DATA_CENTER,hos_ip=${HOS_ENDPOINTS[$DATA_CENTER]}" ;;
+    *) echo "Invalid diagnostic type"; show_help; exit 1 ;;
+  esac
+fi
+
+if $clear_flag; then
+  clear_data
+fi
+
+if $all_flag; then
+ 
+  for data_center_var in "${!KAFKA_SERVERS[@]}"; do
+     generate_logs "$data_center_var" "${KAFKA_SERVERS[$data_center_var]}"
+     generate_metrics "$data_center_var" "${KAFKA_SERVERS[$data_center_var]}"
+     generate_files "$data_center_var" "${KAFKA_SERVERS[$data_center_var]}"
+     echo "Wait 30 Seconds..."
+     sleep 30
+     run_diagnostic "logs" "data_center=$data_center_var"
+     run_diagnostic "metrics" "data_center=$data_center_var"
+     run_diagnostic "files" "data_center=$data_center_var,hos_ip=${HOS_ENDPOINTS[$data_center_var]}" 
+     clear_data
+ done  
+fi
+
+# Display help message if no arguments are passed
+if ! $generate_flag && ! $diagnostic_flag && ! $clear_flag && ! $all_flag; then
+  show_help
+  exit 1
+fi
+
+log_message "E2E test execution completed."
diff --git a/24.05/bin/function_test.sh b/24.05/bin/function_test.sh
new file mode 100755
index 0000000..772e9f3
--- /dev/null
+++ b/24.05/bin/function_test.sh
@@ -0,0 +1,118 @@
+#!/bin/bash
+
+show_help() {
+  echo "
+Usage: $(basename "$0") [-f FOLDER_NUMBER] [-a] [-v] [-e]
+
+Options:
+  -f FOLDER_NUMBER  Specify a single folder to run from a collection:
+                    1. Query
+                    2. Dataset
+                    3. Database
+                    4. Util
+                    5. Troubleshooting
+                    6. HOS
+                    7. Knowledge Base File
+  -a                Run all options [1-7]
+  -v                Enable verbose reporting
+  -e                Enable emojitrain reporting
+  -h, --help        Show this help message and exit
+"
+}
+
+BASE_DIR="$(dirname "$(pwd)")"
+CONFIG_PATH="$BASE_DIR/config"
+FOLDER=""
+ALL=false
+VERBOSE_FLAG=false
+EMOJITRAIN_FLAG=false
+
+# Flag to check if any valid option is provided
+VALID_OPTION_PROVIDED=false
+
+# Parse command-line arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -f)
+      VALID_OPTION_PROVIDED=true
+      if [[ -n $2 && $2 =~ ^[1-7]$ ]]; then
+        case $2 in
+          1) FOLDER="--folder Query" ;;
+          2) FOLDER="--folder Dataset" ;;
+          3) FOLDER="--folder Database" ;;
+          4) FOLDER="--folder Util" ;;
+          5) FOLDER="--folder Troubleshooting" ;;
+          6) FOLDER="--folder HOS" ;;
+          7) FOLDER="--folder Knowledge Base File" ;;
+        esac
+        shift 2
+      else
+        echo "Error: Invalid folder number."
+        show_help
+        exit 1
+      fi
+      ;;
+    -a)
+      VALID_OPTION_PROVIDED=true
+      ALL=true
+      shift
+      ;;
+    -v)
+      VERBOSE_FLAG=true
+      shift
+      ;;
+    -e)
+      EMOJITRAIN_FLAG=true
+      shift
+      ;;
+    -h|--help)
+      show_help
+      exit 0
+      ;;
+    *)
+      echo "Error: Unknown option $1"
+      show_help
+      exit 1
+      ;;
+  esac
+done
+
+if ! $VALID_OPTION_PROVIDED; then
+  show_help
+  exit 1
+fi
+
+if $ALL; then
+  FOLDER=""
+fi
+
+NEW_MAN_OPTS=(
+  "$CONFIG_PATH/tsg-olap-function-test-collection.json"
+  -n 1
+  -e "$CONFIG_PATH/environment.json"
+  -g "$CONFIG_PATH/globals.json"
+  --delay-request 200
+  --timeout-script 10000
+  --timeout-request 300000
+  --timeout 3600000
+  --insecure
+)
+
+# Add verbose reporting  option if enabled
+if $VERBOSE_FLAG; then
+  NEW_MAN_OPTS+=(--verbose)
+fi
+
+# Add folder option if set
+if [[ -n $FOLDER ]]; then
+  NEW_MAN_OPTS+=($FOLDER)
+fi
+
+# Add emojitrain reporting option if enabled
+if $EMOJITRAIN_FLAG; then
+  NEW_MAN_OPTS+=(-r emojitrain)
+fi
+
+# Run the newman command
+newman run "${NEW_MAN_OPTS[@]}"
+
diff --git a/24.05/environment.json b/24.05/config/environment.json
index dd885ce..dd885ce 100644
--- a/24.05/environment.json
+++ b/24.05/config/environment.json
diff --git a/24.05/config/globals.json b/24.05/config/globals.json
new file mode 100644
index 0000000..0397c3f
--- /dev/null
+++ b/24.05/config/globals.json
@@ -0,0 +1,39 @@
+{
+	"id": "0083244f-f7da-4ec8-8c09-317c1121d3ad",
+	"values": [
+		{
+			"key": "start_time",
+			"value": "",
+			"type": "any",
+			"enabled": true
+		},
+		{
+			"key": "end_time",
+			"value": "",
+			"type": "any",
+			"enabled": true
+		},
+		{
+			"key": "domain",
+			"value": "",
+			"type": "any",
+			"enabled": true
+		},
+		{
+			"key": "client_ip",
+			"value": "",
+			"type": "any",
+			"enabled": true
+		},
+		{
+			"key": "server_ip",
+			"value": "",
+			"type": "any",
+			"enabled": true
+		}
+	],
+	"name": "Globals",
+	"_postman_variable_scope": "globals",
+	"_postman_exported_at": "2024-05-21T02:39:11.566Z",
+	"_postman_exported_using": "Postman/11.1.3"
+}
\ No newline at end of file
diff --git a/24.05/tsg-olap-e2e-test-collection.json b/24.05/config/tsg-olap-e2e-test-collection.json
index 4255ec2..4255ec2 100644
--- a/24.05/tsg-olap-e2e-test-collection.json
+++ b/24.05/config/tsg-olap-e2e-test-collection.json
diff --git a/24.05/tsg-olap-function-test-collection.json b/24.05/config/tsg-olap-function-test-collection.json
index d1cd19a..d1cd19a 100644
--- a/24.05/tsg-olap-function-test-collection.json
+++ b/24.05/config/tsg-olap-function-test-collection.json
diff --git a/24.05/e2e-mockdata-generator/config.properties b/24.05/e2e-mockdata-generator/config.properties
index 6c07568..9080c00 100644
--- a/24.05/e2e-mockdata-generator/config.properties
+++ b/24.05/e2e-mockdata-generator/config.properties
@@ -1,5 +1,3 @@
-#####9092-Plaintext 9094-SASL
-kafka.server=192.168.44.11:9092
 #####kafka SASL username/password
 #kafka.user=admin
-#kafka.pin=galaxy2019
+#kafka.pin=galaxy2019
\ No newline at end of file
diff --git a/24.05/e2e-mockdata-generator/datasets/logs/proxy_event.dat b/24.05/e2e-mockdata-generator/datasets/logs/proxy_event.dat
index c487511..8f8ea89 100644
--- a/24.05/e2e-mockdata-generator/datasets/logs/proxy_event.dat
+++ b/24.05/e2e-mockdata-generator/datasets/logs/proxy_event.dat
@@ -1 +1 @@
-{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240001,"decoded_as":"HTTP","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.1","server_ip":"192.0.2.1","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.HTTP","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"http_version":"http1","http_request_line":"GET www.google.com/ HTTP/1.1","http_response_line":"HTTP/1.1 200 OK","http_status_code":200,"http_url":"www.google.com/","http_host":"www.google.com","http_cookie":"NID=513","http_user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; WOW64; Trident/5.0)","http_response_content_type":"text/html; charset=UTF-8","proxy_rule_list":[4450],"proxy_action":"insert","http_action_file_size":35}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240001,"decoded_as":"HTTP","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.1","server_ip":"192.0.2.1","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.HTTP","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"http_version":"http1","http_request_line":"GET www.google.com/ HTTP/1.1","http_response_line":"HTTP/1.1 200 OK","http_status_code":200,"http_url":"www.google.com/","http_host":"www.google.com","http_cookie":"NID=513","http_user_agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; WOW64; Trident/5.0)","http_response_content_type":"text/html; charset=UTF-8","proxy_rule_list":[4450],"proxy_action":"insert","http_action_file_size":35}
diff --git a/24.05/e2e-mockdata-generator/datasets/logs/session_record.dat b/24.05/e2e-mockdata-generator/datasets/logs/session_record.dat
index 1821406..1db77e7 100644
--- a/24.05/e2e-mockdata-generator/datasets/logs/session_record.dat
+++ b/24.05/e2e-mockdata-generator/datasets/logs/session_record.dat
@@ -1,12 +1,7 @@
-{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240001,"decoded_as":"BASE","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.1","server_ip":"192.0.2.1","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]"}
-{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240002,"decoded_as":"HTTP","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.2","server_ip":"192.0.2.2","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.HTTP","server_fqdn":"static-pcs-sdk-server.test.com","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","packet_capture_file":"123e4567-e89b-12d3-a456-426614174001","http_request_body":"123e4567-e89b-12d3-a456-426614174002","http_response_body":"123e4567-e89b-12d3-a456-426614174003","http_version":"http1","http_request_line":"POST /a HTTP/1.1","http_user_agent":"WinHttpClient","http_request_content_length":0,"http_host":"static-pcs-sdk-server.test.com","http_url":"static-pcs-sdk-server.test.com/a","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"application/json;charset=UTF-8","http_response_content_length":0,"http_response_latency_ms":0,"http_session_duration_ms":0,"http_sequence":1}
-{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240003,"decoded_as":"SSL","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.3","server_ip":"192.0.2.3","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.ssl","server_fqdn":"storeedgefd.dsx.mp.microsoft.com","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","ssl_ja3_hash":"","ssl_esni_flag":0,"ssl_ech_flag":0,"ssl_sni":"storeedgefd.dsx.mp.microsoft.com","ssl_ja3s_hash":"","ssl_version":"v3","ssl_cn":"sfdataservice.microsoft.com","ssl_cert_issuer":"","ssl_cert_subject":"","ssl_san":"","ssl_handshake_latency_ms":0}
-{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240004,"decoded_as":"DNS","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.4","server_ip":"192.0.2.4","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.dns","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","dns_response_latency_ms":0,"dns_qr":1,"dns_aa":0,"dns_message_id":47610,"dns_opcode":0,"dns_ra":1,"dns_rcode":0,"dns_rd":1,"dns_tc":0,"dns_qdcount":1,"dns_ancount":1,"dns_nscount":6,"dns_arcount":13,"dns_qname":"","dns_qtype":1,"dns_qclass":1}
-{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240005,"decoded_as":"QUIC","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.5","server_ip":"192.0.2.5","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.quic","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","quic_version":"IETF QUIC RFC9000"}
-{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240006,"decoded_as":"MAIL","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.6","server_ip":"192.0.2.6","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.mail.imap","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","mail_protocol_type":"IMAP","mail_eml_file":"123e4567-e89b-12d3-a456-426614174004"}
-{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240007,"decoded_as":"SSH","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.7","server_ip":"192.0.2.7","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.ssh","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"tsg_olap\"},{\"tag\":\"device_group\",\"value\":\"tsg_olap\"}]}","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","ssh_server_version":"SSH-2.0-OpenSSH_7.4","ssh_client_version":"SSH-2.0-OpenSSH_9.0","ssh_version":2,"ssh_hassh":"0","ssh_kex_alg":"curve25519-sha256","ssh_host_key_alg":"ssh-ed25519","ssh_cipher_alg":"[email protected]","ssh_mac_alg":"[email protected]","ssh_compression_alg":"none","ssh_host_key":"0"}
-
-
-
-
-
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240001,"decoded_as":"BASE","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.1","server_ip":"192.0.2.1","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]"}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240002,"decoded_as":"HTTP","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.2","server_ip":"192.0.2.2","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.HTTP","server_fqdn":"static-pcs-sdk-server.test.com","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","packet_capture_file":"123e4567-e89b-12d3-a456-426614174001","http_request_body":"123e4567-e89b-12d3-a456-426614174002","http_response_body":"123e4567-e89b-12d3-a456-426614174003","http_version":"http1","http_request_line":"POST /a HTTP/1.1","http_user_agent":"WinHttpClient","http_request_content_length":0,"http_host":"static-pcs-sdk-server.test.com","http_url":"static-pcs-sdk-server.test.com/a","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"application/json;charset=UTF-8","http_response_content_length":0,"http_response_latency_ms":0,"http_session_duration_ms":0,"http_sequence":1}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240003,"decoded_as":"SSL","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.3","server_ip":"192.0.2.3","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.ssl","server_fqdn":"storeedgefd.dsx.mp.microsoft.com","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","ssl_ja3_hash":"","ssl_esni_flag":0,"ssl_ech_flag":0,"ssl_sni":"storeedgefd.dsx.mp.microsoft.com","ssl_ja3s_hash":"","ssl_version":"v3","ssl_cn":"sfdataservice.microsoft.com","ssl_cert_issuer":"","ssl_cert_subject":"","ssl_san":"","ssl_handshake_latency_ms":0}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240004,"decoded_as":"DNS","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.4","server_ip":"192.0.2.4","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.dns","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","dns_response_latency_ms":0,"dns_qr":1,"dns_aa":0,"dns_message_id":47610,"dns_opcode":0,"dns_ra":1,"dns_rcode":0,"dns_rd":1,"dns_tc":0,"dns_qdcount":1,"dns_ancount":1,"dns_nscount":6,"dns_arcount":13,"dns_qname":"","dns_qtype":1,"dns_qclass":1}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240005,"decoded_as":"QUIC","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.5","server_ip":"192.0.2.5","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.quic","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","quic_version":"IETF QUIC RFC9000"}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240006,"decoded_as":"MAIL","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.6","server_ip":"192.0.2.6","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.mail.imap","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","mail_protocol_type":"IMAP","mail_eml_file":"123e4567-e89b-12d3-a456-426614174004"}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240007,"decoded_as":"SSH","ip_protocol":"tcp","address_type":4,"client_ip":"10.0.0.7","server_ip":"192.0.2.7","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.TCP.ssh","server_fqdn":"","out_src_mac":"00:1A:2B:3C:4D:5E","out_dest_mac":"5E:4D:3C:2B:1A:00","start_timestamp_ms":946681200000,"end_timestamp_ms":946681200000,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"tcp_c2s_ip_fragments":0,"tcp_s2c_ip_fragments":0,"tcp_c2s_rtx_pkts":0,"tcp_c2s_rtx_bytes":0,"tcp_s2c_rtx_pkts":0,"tcp_s2c_rtx_bytes":0,"tcp_c2s_o3_pkts":0,"tcp_s2c_o3_pkts":0,"tcp_c2s_lost_bytes":0,"tcp_s2c_lost_bytes":0,"flags":0,"flags_identify_info":[1,1],"fqdn_category_list":[0],"monitor_rule_list":[0],"security_rule_list":[0],"sc_rule_list":[0],"shaping_rule_list":[0],"proxy_rule_list":[0],"statistics_rule_list":[0],"monitor_mirrored_pkts":0,"monitor_mirrored_bytes":0,"client_os_desc":"Windows","server_os_desc":"Linux","dup_traffic_flag":0,"sc_rsp_raw":[0],"encapsulation":"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"00:1A:2B:3C:4D:5E\",\"destination_mac\":\"5E:4D:3C:2B:1A:00\"}]","ssh_server_version":"SSH-2.0-OpenSSH_7.4","ssh_client_version":"SSH-2.0-OpenSSH_9.0","ssh_version":2,"ssh_hassh":"0","ssh_kex_alg":"curve25519-sha256","ssh_host_key_alg":"ssh-ed25519","ssh_cipher_alg":"[email protected]","ssh_mac_alg":"[email protected]","ssh_compression_alg":"none","ssh_host_key":"0"}
diff --git a/24.05/e2e-mockdata-generator/e2e-mockdata-generator.jar b/24.05/e2e-mockdata-generator/e2e-mockdata-generator.jar
index b9c6127..8e6f26c 100644
--- a/24.05/e2e-mockdata-generator/e2e-mockdata-generator.jar
+++ b/24.05/e2e-mockdata-generator/e2e-mockdata-generator.jar