summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorqidaijie <[email protected]>2023-02-27 11:52:48 +0800
committerqidaijie <[email protected]>2023-02-27 11:52:48 +0800
commitead83f23ad9f754b3ebdd155ab7aa5cb1d3cfe09 (patch)
tree844686ca454e9e8daba49537d15d9363113765f5
parentd0be47194abf9f035da37ffb3bf6af130c19ef05 (diff)
提交POC测试时log-stream和file-sync程序配置P19-POC
Diffstat
-rw-r--r--file-sync-service/volumes/p19-file-sync-service/config/application.yml21
-rw-r--r--file-sync-service/volumes/p19-file-sync-service/config/log4j2-dev.xml56
-rw-r--r--file-sync-service/workspace/p19-file-sync-service/config/p19-file-sync-service.yaml38
-rw-r--r--file-sync-service/workspace/p19-file-sync-service/docker-compose.yml15
-rw-r--r--file-sync-service/workspace/p19-file-sync-service/push_config.sh55
-rw-r--r--log-stream/.gitkeep0
-rw-r--r--log-stream/config/PROXY-EVENT-PERCENT78
-rw-r--r--log-stream/config/SECURITY-EVENT-PERCENT78
-rw-r--r--log-stream/config/SESSION-RECORD-PERCENT78
-rw-r--r--log-stream/push_config.sh55
-rw-r--r--log-stream/schema/proxy_event.json1543
-rw-r--r--log-stream/schema/security_event.json2477
-rw-r--r--log-stream/schema/session_record.json2465
-rw-r--r--log-stream/start.sh40
-rw-r--r--log-stream/stop.sh29
15 files changed, 7028 insertions, 0 deletions
diff --git a/file-sync-service/volumes/p19-file-sync-service/config/application.yml b/file-sync-service/volumes/p19-file-sync-service/config/application.yml
new file mode 100644
index 0000000..71570b2
--- /dev/null
+++ b/file-sync-service/volumes/p19-file-sync-service/config/application.yml
@@ -0,0 +1,21 @@
+nacos:
+ config:
+ type: yaml
+ server-addr: 10.3.60.2:8848
+ namespace: prod
+ data-id: p19-file-sync-service.yaml
+ auto-refresh: true
+ group: Galaxy
+ username: nacos
+ password: nacos
+ bootstrap:
+ enable: true
+ log:
+ enable: true
+ discovery:
+ service-name: hos
+spring:
+ profiles:
+ active: dev
+logging:
+ config: ./config/log4j2-dev.xml
diff --git a/file-sync-service/volumes/p19-file-sync-service/config/log4j2-dev.xml b/file-sync-service/volumes/p19-file-sync-service/config/log4j2-dev.xml
new file mode 100644
index 0000000..76a4e15
--- /dev/null
+++ b/file-sync-service/volumes/p19-file-sync-service/config/log4j2-dev.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+ <!--日志打印相关参数配置-->
+ <Properties>
+ <!--每5M压缩日志文件-->
+ <property name="LOG_SIZE">100M</property>
+ <!--最多产生10个压缩文件-->
+ <property name="LOG_NUMS">5</property>
+ <!--日志打印等级-->
+ <property name="LOG_LEVEL">info</property>
+ <!--日志文件路径-->
+ <property name="LOG_PATH">logs</property>
+ <!--日志文件名称-->
+ <property name="LOG_FILE_NAME">galaxy-fsync-service</property>
+ <!--日志打印格式-->
+ <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ss}] [%p] [Thread:%t] %l %x - %m%n</property>
+ </Properties>
+
+ <appenders>
+ <Console name="consoleSystemOutAppender" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="${LOG_PATTERN}"/>
+ </Console>
+
+ <RollingFile name="rollingFileAllAppender"
+ fileName="${LOG_PATH}/${LOG_FILE_NAME}.log"
+ filePattern="${LOG_PATH}/history/$${date:yyyy-MM-dd}/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz">
+ <PatternLayout pattern="${LOG_PATTERN}"/>
+ <Policies>
+ <SizeBasedTriggeringPolicy size="${LOG_SIZE}"/>
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <Filters>
+ <ThresholdFilter level="all" onMatch="ACCEPT" onMismatch="DENY"/>
+ </Filters>
+ <DefaultRolloverStrategy max="${LOG_NUMS}">
+ <Delete basePath="${LOG_PATH}/history" maxDepth="1">
+ <IfFileName glob="*.log.gz">
+ <IfLastModified age="90d">
+ <IfAny>
+ <IfAccumulatedFileSize exceeds="200 GB" />
+ </IfAny>
+ </IfLastModified>
+ </IfFileName>
+ </Delete>
+ </DefaultRolloverStrategy>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="${LOG_LEVEL}">
+ <appender-ref ref="consoleSystemOutAppender"/>
+ <appender-ref ref="rollingFileAllAppender"/>
+ </root>
+ </loggers>
+</configuration>
diff --git a/file-sync-service/workspace/p19-file-sync-service/config/p19-file-sync-service.yaml b/file-sync-service/workspace/p19-file-sync-service/config/p19-file-sync-service.yaml
new file mode 100644
index 0000000..0e2fdd8
--- /dev/null
+++ b/file-sync-service/workspace/p19-file-sync-service/config/p19-file-sync-service.yaml
@@ -0,0 +1,38 @@
+spring:
+ application:
+ name: P19FileSyncServiceApplication
+server:
+ port: 8189
+#最大线程数
+thread:
+ maxSize: 1000
+http:
+ connectTimeout: 10000
+ connectionRequestTimeout: 10000
+ defaultMaxPerRoute: 1000
+ maxTotal: 2000
+ socketTimeout: 180000
+ staleConnectionCheckEnabled: true
+kafka:
+ consumer:
+ #kafka地址
+ servers: 10.3.60.2:9094,10.3.60.3:9094,10.3.60.4:9094
+ topic: FILE-TO-OSS
+ group_id: 1
+ auto_commit_interval: 100
+ #latest，消费最新消息，保证每个组一个消费者消费同一条消息；earliest，从头开始消费消息
+ auto_offset_reset: latest
+ #是否是批量消费
+ batch_listener: true
+ #批量消费条数
+ pool_record: 1000
+ #并发数，设置和topic分区数一致
+ concurrency: 3
+ enable_auto_commit: false
+ poll_timeout: 1500
+ session_timeout: 60000
+ #开启验证的用户名和密码
+ sasl_password: galaxy2019
+ sasl_username: admin
+logging:
+ config: ./config/log4j2-dev.xml
diff --git a/file-sync-service/workspace/p19-file-sync-service/docker-compose.yml b/file-sync-service/workspace/p19-file-sync-service/docker-compose.yml
new file mode 100644
index 0000000..5f4bdfd
--- /dev/null
+++ b/file-sync-service/workspace/p19-file-sync-service/docker-compose.yml
@@ -0,0 +1,15 @@
+version: '2'
+
+services:
+ galaxy-hos-service:
+ image: p19-file-sync-service:21.12.01
+ container_name: p19-file-sync-service
+ environment:
+ JAVA_OPTS: "-Xms2g -Xmx2g -Xmn256m -Xss256k -XX:MetaspaceSize=256m -XX:MaxPermSize=256m -XX:SurvivorRatio=2 -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -XX:MaxTenuringThreshold=15 -XX:+UseCMSCompactAtFullCollection -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+ volumes:
+ - "/home/tsg/olap/galaxy/volumes/p19-file-sync-service/config:/home/tsg/galaxy/p19-file-sync-service/config"
+ - "/home/tsg/olap/galaxy/volumes/p19-file-sync-service/logs:/home/tsg/galaxy/p19-file-sync-service/logs"
+ - "/etc/localtime:/etc/localtime:ro"
+ - "/etc/timezone:/etc/timezone:ro"
+ restart: always
+ network_mode: "host"
diff --git a/file-sync-service/workspace/p19-file-sync-service/push_config.sh b/file-sync-service/workspace/p19-file-sync-service/push_config.sh
new file mode 100644
index 0000000..25c32b3
--- /dev/null
+++ b/file-sync-service/workspace/p19-file-sync-service/push_config.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+nacos_server=$1
+config_path_root=$2
+app_name=p19-file-sync-service
+tenant=prod
+
+nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy"
+
+shell_resu=1
+
+#1.获取config 路径
+config_path_list=$(readlink -f $config_path_root)
+
+function push(){
+if [ "$suffix" = "yaml" ]||[ "$suffix" = "json" ]||[ "$suffix" = "text" ]||[ "$suffix" = "properties" ];then
+ result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content="`cat $config_path_file`")
+ if [[ $result -eq '200' ]];then
+ check=`curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id"`
+ if [[ $check -ne '200' ]];then
+ shell_resu=0
+ fi
+ fi
+fi
+}
+
+
+function pushConfig(){
+#2.app文件夹文件
+for config_path in $config_path_list
+do
+if [ -d $config_path ];then
+
+ for file in `ls $config_path`
+ do
+ config_path_file=$config_path/$file
+ if [ -f $config_path_file ];then
+ data_id=${config_path_file##*/}
+ suffix=${config_path_file##*.}
+ if [ "$suffix" = "sql" ];then
+ suffix="text"
+ fi
+
+ if [ "$suffix" = "yml" ];then
+ suffix="yaml"
+ fi
+#3.提交config配置
+ push
+ fi
+ done
+fi
+done
+}
+pushConfig
+echo $shell_resu
diff --git a/log-stream/.gitkeep b/log-stream/.gitkeep
deleted file mode 100644
index e69de29..0000000
--- a/log-stream/.gitkeep
+++ /dev/null
diff --git a/log-stream/config/PROXY-EVENT-PERCENT b/log-stream/config/PROXY-EVENT-PERCENT
new file mode 100644
index 0000000..dfcc6e3
--- /dev/null
+++ b/log-stream/config/PROXY-EVENT-PERCENT
@@ -0,0 +1,78 @@
+#--------------------------------地址配置------------------------------#
+#管理kafka地址
+source.kafka.servers=10.3.60.2:9094,10.3.60.3:9094,10.3.60.4:9094
+
+#百分点输出kafka地址
+percent.sink.kafka.servers=10.3.45.126:6667,10.3.45.127:6667,10.3.45.128:6667
+
+#文件源数据topic输出kafka地址
+file.data.sink.kafka.servers=10.3.60.2:9094,10.3.60.3:9094,10.3.60.4:9094
+
+#zookeeper 地址 用于配置log_id
+zookeeper.servers=10.3.60.2:2181,10.3.60.3:2181,10.3.60.4:2181
+
+#hbase zookeeper地址 用于连接HBase
+hbase.zookeeper.servers=10.3.60.2:2181,10.3.60.3:2181,10.3.60.4:2181
+
+#--------------------------------HTTP/定位库------------------------------#
+#定位库地址
+tools.library=/home/tsg/olap/topology/dat/
+
+#--------------------------------nacos配置------------------------------#
+#nacos 地址
+nacos.server=10.3.60.3:8848
+
+#nacos namespace
+nacos.schema.namespace=dev
+
+#nacos topology_common_config.properties namespace
+nacos.common.namespace=dev
+
+#nacos data id
+nacos.data.id=proxy_event.json
+
+#------------------------------------OOS配置------------------------------------#
+#oos地址
+oos.servers=10.3.45.124:8057
+
+#--------------------------------Kafka消费/生产配置------------------------------#
+
+#kafka 接收数据topic
+source.kafka.topic=PROXY-EVENT
+
+#百分点对应的topic
+percent.kafka.topic=PROXY-EVENT
+
+#文件源数据topic
+file.data.kafka.topic=FILE-TO-OSS
+
+#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
+group.id=proxy-event-percent-1
+
+#--------------------------------topology配置------------------------------#
+
+#consumer 并行度
+source.parallelism=3
+
+#转换函数并行度
+transform.parallelism=3
+
+#percent producer 并行度
+percent.sink.parallelism=3
+
+#filedata producer 并行度
+file.data.sink.parallelism=3
+
+#数据中心，取值范围(0-31)
+data.center.id.num=31
+
+#hbase 更新时间，如填写0则不更新缓存
+hbase.tick.tuple.freq.secs=180
+
+#--------------------------------默认值配置------------------------------#
+
+#0不需要补全原样输出日志，1需要补全
+log.need.complete=1
+
+#生产者压缩模式 none or snappy
+producer.kafka.compression.type=none
diff --git a/log-stream/config/SECURITY-EVENT-PERCENT b/log-stream/config/SECURITY-EVENT-PERCENT
new file mode 100644
index 0000000..227b5af
--- /dev/null
+++ b/log-stream/config/SECURITY-EVENT-PERCENT
@@ -0,0 +1,78 @@
+#--------------------------------地址配置------------------------------#
+#管理kafka地址
+source.kafka.servers=10.3.60.2:9094,10.3.60.3:9094,10.3.60.4:9094
+
+#百分点输出kafka地址
+percent.sink.kafka.servers=10.3.45.126:6667,10.3.45.127:6667,10.3.45.128:6667
+
+#文件源数据topic输出kafka地址
+file.data.sink.kafka.servers=10.3.60.2:9094,10.3.60.3:9094,10.3.60.4:9094
+
+#zookeeper 地址 用于配置log_id
+zookeeper.servers=10.3.60.2:2181,10.3.60.3:2181,10.3.60.4:2181
+
+#hbase zookeeper地址 用于连接HBase
+hbase.zookeeper.servers=10.3.60.2:2181,10.3.60.3:2181,10.3.60.4:2181
+
+#--------------------------------HTTP/定位库------------------------------#
+#定位库地址
+tools.library=/home/tsg/olap/topology/dat/
+
+#--------------------------------nacos配置------------------------------#
+#nacos 地址
+nacos.server=10.3.60.3:8848
+
+#nacos namespace
+nacos.schema.namespace=dev
+
+#nacos topology_common_config.properties namespace
+nacos.common.namespace=dev
+
+#nacos data id
+nacos.data.id=security_event.json
+
+#------------------------------------OOS配置------------------------------------#
+#oos地址
+oos.servers=10.3.45.124:8057
+
+#--------------------------------Kafka消费/生产配置------------------------------#
+
+#kafka 接收数据topic
+source.kafka.topic=SECURITY-EVENT
+
+#百分点对应的topic
+percent.kafka.topic=SECURITY-EVENT
+
+#文件源数据topic
+file.data.kafka.topic=FILE-TO-OSS
+
+#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
+group.id=security-event-percent-1
+
+#--------------------------------topology配置------------------------------#
+
+#consumer 并行度
+source.parallelism=3
+
+#转换函数并行度
+transform.parallelism=3
+
+#percent producer 并行度
+percent.sink.parallelism=3
+
+#filedata producer 并行度
+file.data.sink.parallelism=3
+
+#数据中心，取值范围(0-31)
+data.center.id.num=30
+
+#hbase 更新时间，如填写0则不更新缓存
+hbase.tick.tuple.freq.secs=180
+
+#--------------------------------默认值配置------------------------------#
+
+#0不需要补全原样输出日志，1需要补全
+log.need.complete=1
+
+#生产者压缩模式 none or snappy
+producer.kafka.compression.type=none
diff --git a/log-stream/config/SESSION-RECORD-PERCENT b/log-stream/config/SESSION-RECORD-PERCENT
new file mode 100644
index 0000000..706c8d2
--- /dev/null
+++ b/log-stream/config/SESSION-RECORD-PERCENT
@@ -0,0 +1,78 @@
+#--------------------------------地址配置------------------------------#
+#管理kafka地址
+source.kafka.servers=10.3.60.2:9094,10.3.60.3:9094,10.3.60.4:9094
+
+#百分点输出kafka地址
+percent.sink.kafka.servers=10.3.45.126:6667,10.3.45.127:6667,10.3.45.128:6667
+
+#文件源数据topic输出kafka地址
+file.data.sink.kafka.servers=10.3.60.2:9094,10.3.60.3:9094,10.3.60.4:9094
+
+#zookeeper 地址 用于配置log_id
+zookeeper.servers=10.3.60.2:2181,10.3.60.3:2181,10.3.60.4:2181
+
+#hbase zookeeper地址 用于连接HBase
+hbase.zookeeper.servers=10.3.60.2:2181,10.3.60.3:2181,10.3.60.4:2181
+
+#--------------------------------HTTP/定位库------------------------------#
+#定位库地址
+tools.library=/home/tsg/olap/topology/dat/
+
+#--------------------------------nacos配置------------------------------#
+#nacos 地址
+nacos.server=10.3.60.3:8848
+
+#nacos namespace
+nacos.schema.namespace=dev
+
+#nacos topology_common_config.properties namespace
+nacos.common.namespace=dev
+
+#nacos data id
+nacos.data.id=session_record.json
+
+#------------------------------------OOS配置------------------------------------#
+#oos地址
+oos.servers=10.3.45.124:8057
+
+#--------------------------------Kafka消费/生产配置------------------------------#
+
+#kafka 接收数据topic
+source.kafka.topic=SESSION-RECORD
+
+#百分点对应的topic
+percent.kafka.topic=SESSION-RECORD
+
+#文件源数据topic
+file.data.kafka.topic=FILE-TO-OSS
+
+#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
+group.id=proxy-event-percent-1
+
+#--------------------------------topology配置------------------------------#
+
+#consumer 并行度
+source.parallelism=9
+
+#转换函数并行度
+transform.parallelism=9
+
+#percent producer 并行度
+percent.sink.parallelism=9
+
+#filedata producer 并行度
+file.data.sink.parallelism=9
+
+#数据中心，取值范围(0-31)
+data.center.id.num=29
+
+#hbase 更新时间，如填写0则不更新缓存
+hbase.tick.tuple.freq.secs=180
+
+#--------------------------------默认值配置------------------------------#
+
+#0不需要补全原样输出日志，1需要补全
+log.need.complete=1
+
+#生产者压缩模式 none or snappy
+producer.kafka.compression.type=none
diff --git a/log-stream/push_config.sh b/log-stream/push_config.sh
new file mode 100644
index 0000000..66853ec
--- /dev/null
+++ b/log-stream/push_config.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+nacos_server=$1
+config_path_root=$2
+app_name=log-stream
+tenant=dev
+
+nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy"
+
+shell_resu=1
+
+#1.获取config 路径
+config_path_list=$(readlink -f $config_path_root)
+
+function push(){
+if [ "$suffix" = "yaml" ]||[ "$suffix" = "json" ]||[ "$suffix" = "text" ]||[ "$suffix" = "properties" ];then
+ result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content="`cat $config_path_file`")
+ if [[ $result -eq '200' ]];then
+ check=`curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id"`
+ if [[ $check -ne '200' ]];then
+ shell_resu=0
+ fi
+ fi
+fi
+}
+
+
+function pushConfig(){
+#2.app文件夹文件
+for config_path in $config_path_list
+do
+if [ -d $config_path ];then
+
+ for file in `ls $config_path`
+ do
+ config_path_file=$config_path/$file
+ if [ -f $config_path_file ];then
+ data_id=${config_path_file##*/}
+ suffix=${config_path_file##*.}
+ if [ "$suffix" = "sql" ];then
+ suffix="text"
+ fi
+
+ if [ "$suffix" = "yml" ];then
+ suffix="yaml"
+ fi
+#3.提交config配置
+ push
+ fi
+ done
+fi
+done
+}
+pushConfig
+echo $shell_resu
diff --git a/log-stream/schema/proxy_event.json b/log-stream/schema/proxy_event.json
new file mode 100644
index 0000000..b151d6c
--- /dev/null
+++ b/log-stream/schema/proxy_event.json
@@ -0,0 +1,1543 @@
+{
+ "type": "record",
+ "name": "proxy_event",
+ "namespace": "tsg_galaxy_v3",
+ "doc": {
+ "primary_key": "common_log_id",
+ "partition_key": "common_recv_time",
+ "functions": {
+ "$ref": "public_schema_info.json#/functions"
+ },
+ "schema_query": {
+ "dimensions": [
+ "common_server_ip",
+ "common_client_ip",
+ "common_internal_ip",
+ "common_external_ip",
+ "common_policy_id",
+ "common_sub_action",
+ "common_sled_ip",
+ "common_device_id",
+ "common_client_location",
+ "common_server_location",
+ "common_subscriber_id",
+ "common_client_port",
+ "common_server_port",
+ "common_schema_type",
+ "common_data_center",
+ "common_device_group",
+ "common_app_behavior",
+ "common_client_asn",
+ "common_server_asn",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number",
+ "http_host",
+ "http_domain",
+ "http_url",
+ "http_cookie",
+ "http_referer",
+ "http_user_agent",
+ "doh_host",
+ "doh_qname"
+ ],
+ "metrics": [
+ "common_server_ip",
+ "common_client_ip",
+ "common_internal_ip",
+ "common_external_ip",
+ "common_subscriber_id",
+ "common_sled_ip",
+ "common_device_id",
+ "common_sessions",
+ "common_c2s_byte_num",
+ "common_s2c_byte_num",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number",
+ "http_host",
+ "http_domain",
+ "http_url",
+ "http_cookie",
+ "http_referer",
+ "http_user_agent",
+ "doh_host",
+ "doh_qname"
+ ],
+ "filters": [
+ "common_policy_id",
+ "common_sub_action",
+ "common_address_type",
+ "common_server_ip",
+ "common_client_ip",
+ "common_internal_ip",
+ "common_external_ip",
+ "common_client_port",
+ "common_server_port",
+ "common_client_location",
+ "common_server_location",
+ "common_subscriber_id",
+ "common_l4_protocol",
+ "common_data_center",
+ "common_device_group",
+ "common_app_behavior",
+ "common_sled_ip",
+ "common_device_id",
+ "common_client_asn",
+ "common_server_asn",
+ "common_direction",
+ "common_schema_type",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number",
+ "http_host",
+ "http_domain",
+ "http_url",
+ "http_cookie",
+ "http_referer",
+ "http_user_agent",
+ "http_request_content_type",
+ "http_response_content_type",
+ "doh_host",
+ "doh_qname"
+ ],
+ "references": {
+ "$ref": "public_schema_info.json#/schema_query/references"
+ },
+ "details": {
+ "general": [
+ "common_recv_time",
+ "common_log_id",
+ "common_stream_trace_id",
+ "common_direction",
+ "common_stream_dir",
+ "common_start_time",
+ "common_end_time",
+ "common_con_duration_ms",
+ "common_establish_latency_ms",
+ "common_processing_time",
+ "common_ingestion_time",
+ "common_entrance_id",
+ "common_device_id",
+ "common_egress_link_id",
+ "common_ingress_link_id",
+ "common_isp",
+ "common_data_center",
+ "common_device_group",
+ "common_sled_ip"
+ ],
+ "action": [
+ "common_action",
+ "common_sub_action",
+ "common_policy_id",
+ "common_user_tags",
+ "common_user_region"
+ ],
+ "source": [
+ "common_client_ip",
+ "common_internal_ip",
+ "common_client_port",
+ "common_client_location",
+ "common_client_asn",
+ "common_subscriber_id",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number"
+ ],
+ "destination": [
+ "common_server_ip",
+ "common_external_ip",
+ "common_server_port",
+ "common_server_location",
+ "common_server_asn"
+ ],
+ "application": [
+ "common_app_id",
+ "common_userdefine_app_name",
+ "common_app_label",
+ "common_app_surrogate_id",
+ "common_l7_protocol",
+ "common_protocol_label",
+ "common_service_category",
+ "common_service",
+ "common_l4_protocol",
+ "common_app_behavior"
+ ],
+ "transmission": [
+ "common_sessions",
+ "common_c2s_pkt_num",
+ "common_s2c_pkt_num",
+ "common_c2s_byte_num",
+ "common_s2c_byte_num",
+ "common_c2s_pkt_diff",
+ "common_s2c_pkt_diff",
+ "common_c2s_byte_diff",
+ "common_s2c_byte_diff",
+ "common_c2s_ipfrag_num",
+ "common_s2c_ipfrag_num",
+ "common_c2s_tcp_lostlen",
+ "common_s2c_tcp_lostlen",
+ "common_c2s_tcp_unorder_num",
+ "common_s2c_tcp_unorder_num",
+ "common_c2s_pkt_retrans",
+ "common_s2c_pkt_retrans",
+ "common_c2s_byte_retrans",
+ "common_s2c_byte_retrans",
+ "common_first_ttl",
+ "common_tcp_client_isn",
+ "common_tcp_server_isn",
+ "common_mirrored_pkts",
+ "common_mirrored_bytes"
+ ],
+ "other": [
+ "common_address_type",
+ "common_schema_type",
+ "common_device_tag",
+ "common_encapsulation",
+ "common_tunnels",
+ "common_address_list",
+ "common_has_dup_traffic",
+ "common_stream_error",
+ "common_link_info_c2s",
+ "common_link_info_s2c",
+ "common_packet_capture_file"
+ ]
+ }
+ },
+ "schema_type": {
+ "HTTP": {
+ "$ref": "public_schema_info.json#/schema_type/HTTP"
+ },
+ "DoH": {
+ "$ref": "public_schema_info.json#/schema_type/DoH"
+ }
+ },
+ "default_columns": [
+ "common_recv_time",
+ "common_log_id",
+ "common_policy_id",
+ "common_client_ip",
+ "common_server_ip",
+ "common_server_port",
+ "common_sub_action",
+ "common_schema_type"
+ ],
+ "internal_columns": [
+ "common_recv_time",
+ "common_log_id",
+ "common_processing_time",
+ "common_ingestion_time",
+ "common_packet_capture_file",
+ "http_request_body",
+ "http_response_body"
+ ],
+ "tunnel_type": {
+ "$ref": "public_schema_info.json#/tunnel_type"
+ }
+ },
+ "fields": [
+ {
+ "name": "common_recv_time",
+ "label": "Receive Time",
+ "doc": {
+ "constraints": {
+ "type": "timestamp"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_log_id",
+ "label": "Log ID",
+ "doc": {
+ "format": {
+ "functions": "snowflake_id"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_policy_id",
+ "label": "Policy ID",
+ "type": "long"
+ },
+ {
+ "name": "common_subscriber_id",
+ "label": "Subscriber ID",
+ "type": "string"
+ },
+ {
+ "name": "common_imei",
+ "label": "IMEI",
+ "type": "string"
+ },
+ {
+ "name": "common_imsi",
+ "label": "IMSI",
+ "type": "string"
+ },
+ {
+ "name": "common_phone_number",
+ "label": "Phone Number",
+ "type": "string"
+ },
+ {
+ "name": "common_client_ip",
+ "label": "Client IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "geo_asn,radius_match",
+ "appendTo": "common_client_asn,common_subscriber_id"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_internal_ip",
+ "label": "Internal IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "if",
+ "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_client_port",
+ "label": "Client Port",
+ "type": "int"
+ },
+ {
+ "name": "common_l4_protocol",
+ "label": "L4 Protocol",
+ "type": "string"
+ },
+ {
+ "name": "common_address_type",
+ "label": "Address Type",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "4",
+ "value": "ipv4"
+ },
+ {
+ "code": "6",
+ "value": "ipv6"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_server_ip",
+ "label": "Server IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "geo_asn",
+ "appendTo": "common_server_asn"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_server_port",
+ "label": "Server Port",
+ "type": "int"
+ },
+ {
+ "name": "common_external_ip",
+ "label": "External IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "if",
+ "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_action",
+ "label": "Action",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "None"
+ },
+ {
+ "code": "1",
+ "value": "Monitor"
+ },
+ {
+ "code": "2",
+ "value": "Intercept"
+ },
+ {
+ "code": "16",
+ "value": "Deny"
+ },
+ {
+ "code": "48",
+ "value": "Manipulation"
+ },
+ {
+ "code": "128",
+ "value": "Allow"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_direction",
+ "label": "Direction",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "69",
+ "value": "outbound"
+ },
+ {
+ "code": "73",
+ "value": "inbound"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_entrance_id",
+ "label": "Entrance ID",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_sled_ip",
+ "label": "Sled IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_client_location",
+ "label": "Client Location",
+ "type": "string"
+ },
+ {
+ "name": "common_client_asn",
+ "label": "Client ASN",
+ "type": "string"
+ },
+ {
+ "name": "common_server_location",
+ "label": "Server Location",
+ "type": "string"
+ },
+ {
+ "name": "common_server_asn",
+ "label": "Server ASN",
+ "type": "string"
+ },
+ {
+ "name": "common_sessions",
+ "label": "Sessions",
+ "doc": {
+ "visibility": "disabled",
+ "format": {
+ "functions": "set_value",
+ "param": "1"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_pkt_num",
+ "label": "Packets Sent",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_pkt_num",
+ "label": "Packets Received",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_byte_num",
+ "label": "Bytes Sent",
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_byte_num",
+ "label": "Bytes Received",
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_pkt_diff",
+ "label": "Packets Sent(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_pkt_diff",
+ "label": "Packets Received(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_byte_diff",
+ "label": "Bytes Sent(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_byte_diff",
+ "label": "Bytes Received(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_service",
+ "label": "Service",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_schema_type",
+ "label": "Schema Type",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "HTTP",
+ "value": "HTTP"
+ },
+ {
+ "code": "DoH",
+ "value": "DoH"
+ }
+ ]
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_user_tags",
+ "label": "User Tags",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_sub_action",
+ "label": "Sub Action",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "allow",
+ "value": "Allow"
+ },
+ {
+ "code": "deny",
+ "value": "Deny"
+ },
+ {
+ "code": "monitor",
+ "value": "Monitor"
+ },
+ {
+ "code": "replace",
+ "value": "Replace"
+ },
+ {
+ "code": "redirect",
+ "value": "Redirect"
+ },
+ {
+ "code": "insert",
+ "value": "Insert"
+ },
+ {
+ "code": "hijack",
+ "value": "Hijack"
+ },
+ {
+ "code": "edit_element",
+ "value": "Edit Element"
+ }
+ ]
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_user_region",
+ "label": "User Region",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_device_id",
+ "label": "Device ID",
+ "type": "string"
+ },
+ {
+ "name": "common_egress_link_id",
+ "label": "Egress Link ID",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_ingress_link_id",
+ "label": "Ingress Link ID",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_isp",
+ "label": "ISP",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_device_tag",
+ "label": "Device Tag",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_data_center",
+ "label": "Data Center",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "device_tag.json#",
+ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
+ "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
+ },
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_device_group",
+ "label": "Device Group",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "device_tag.json#",
+ "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']",
+ "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_app_behavior",
+ "label": "Application Behavior",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_encapsulation",
+ "label": "Encapsulation",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "public_schema_info.json#/fields/common_encapsulation/data"
+ },
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_app_label",
+ "label": "Application Label",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_tunnels",
+ "label": "Tunnels",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_protocol_label",
+ "label": "Protocol Label",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_app_id",
+ "label": "Application ID",
+ "type": "string",
+ "doc": {
+ "visibility": "disabled"
+ }
+ },
+ {
+ "name": "common_userdefine_app_name",
+ "label": "User Define APP Name",
+ "type": "string",
+ "doc": {
+ "visibility": "disabled"
+ }
+ },
+ {
+ "name": "common_app_surrogate_id",
+ "label": "Surrogate ID",
+ "type": "string",
+ "doc": {
+ "visibility": "disabled"
+ }
+ },
+ {
+ "name": "common_l7_protocol",
+ "label": "L7 Protocol",
+ "type": "string",
+ "doc": {
+ "visibility": "disabled"
+ }
+ },
+ {
+ "name": "common_service_category",
+ "label": "FQDN Category",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "has"
+ },
+ "dict_location": {
+ "path": "/v1/category/dict",
+ "key": "categoryId",
+ "value": "categoryName"
+ }
+ },
+ "type": {
+ "type": "array",
+ "items": "int"
+ }
+ },
+ {
+ "name": "common_start_time",
+ "label": "Start Time",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "timestamp"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_end_time",
+ "label": "End Time",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "timestamp"
+ },
+ "format": {
+ "functions": "get_value",
+ "appendTo": "common_recv_time"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_establish_latency_ms",
+ "label": "Establish Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_con_duration_ms",
+ "label": "Duration(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_stream_dir",
+ "label": "Stream Direction",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "c2s"
+ },
+ {
+ "code": "2",
+ "value": "s2c"
+ },
+ {
+ "code": "3",
+ "value": "double"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_address_list",
+ "label": "Address List",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_has_dup_traffic",
+ "label": "Duplication Traffic",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data"
+ },
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_stream_error",
+ "label": "Stream Error",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_stream_trace_id",
+ "label": "Session ID",
+ "type": "long"
+ },
+ {
+ "name": "common_link_info_c2s",
+ "label": "Link Info(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_link_info_s2c",
+ "label": "Link Info(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_packet_capture_file",
+ "label": "Packet Capture File",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_c2s_ipfrag_num",
+ "label": "Fragmentation Packets(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_ipfrag_num",
+ "label": "Fragmentation Packets(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_tcp_lostlen",
+ "label": "Sequence Gap Loss(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_tcp_lostlen",
+ "label": "Sequence Gap Loss(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_tcp_unorder_num",
+ "label": "Unorder Packets(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_tcp_unorder_num",
+ "label": "Unorder Packets(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_pkt_retrans",
+ "label": "Packet Retransmission(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_pkt_retrans",
+ "label": "Packet Retransmission(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_byte_retrans",
+ "label": "Byte Retransmission(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_byte_retrans",
+ "label": "Byte Retransmission(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_tcp_client_isn",
+ "label": "TCP Client ISN",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_tcp_server_isn",
+ "label": "TCP Server ISN",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_first_ttl",
+ "label": "First TTL",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_processing_time",
+ "label": "Processing Time",
+ "doc": {
+ "constraints": {
+ "type": "timestamp"
+ },
+ "format": {
+ "functions": "current_timestamp"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_ingestion_time",
+ "label": "Ingestion Time",
+ "doc": {
+ "constraints": {
+ "type": "timestamp"
+ },
+ "format": {
+ "functions": "ingestion_time"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_mirrored_pkts",
+ "label": "Mirrored Packets",
+ "type": "long",
+ "doc": {
+ "visibility": "disabled"
+ }
+ },
+ {
+ "name": "common_mirrored_bytes",
+ "label": "Mirrored Bytes",
+ "type": "long",
+ "doc": {
+ "visibility": "disabled"
+ }
+ },
+ {
+ "name": "http_url",
+ "label": "HTTP.URL",
+ "type": "string"
+ },
+ {
+ "name": "http_host",
+ "label": "HTTP.Host",
+ "doc": {
+ "format": {
+ "functions": "sub_domain",
+ "appendTo": "http_domain"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_domain",
+ "label": "HTTP.Domain",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_line",
+ "label": "HTTP.Request Line",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_line",
+ "label": "HTTP.Response Line",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_header",
+ "label": "HTTP.Request Header",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_header",
+ "label": "HTTP.Response Header",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_content",
+ "label": "HTTP.Request Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_content_length",
+ "label": "HTTP.Request Content Length",
+ "type": "string"
+ },
+ {
+ "name": "http_request_content_type",
+ "label": "HTTP.Request Content Type",
+ "type": "string"
+ },
+ {
+ "name": "http_response_content",
+ "label": "HTTP.Response Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_content_length",
+ "label": "HTTP.Response Content Length",
+ "type": "string"
+ },
+ {
+ "name": "http_response_content_type",
+ "label": "HTTP.Response Content Type",
+ "type": "string"
+ },
+ {
+ "name": "http_request_body",
+ "label": "HTTP.Request Body",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_body",
+ "label": "HTTP.Response Body",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_body_key",
+ "label": "HTTP.Request Body Key",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_body_key",
+ "label": "HTTP.Response Body Key",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_proxy_flag",
+ "label": "HTTP.Proxy Flag",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "http_sequence",
+ "label": "HTTP.Sequence",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "http_snapshot",
+ "label": "HTTP.Snapshot",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_cookie",
+ "label": "HTTP.Cookie",
+ "type": "string"
+ },
+ {
+ "name": "http_referer",
+ "label": "HTTP.Referer",
+ "type": "string"
+ },
+ {
+ "name": "http_user_agent",
+ "label": "HTTP.User Agent",
+ "type": "string"
+ },
+ {
+ "name": "http_content_length",
+ "label": "HTTP.Content Length",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_content_type",
+ "label": "HTTP.Content Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_set_cookie",
+ "label": "HTTP.Set Cookie",
+ "type": "string"
+ },
+ {
+ "name": "http_version",
+ "label": "HTTP.Version",
+ "type": "string"
+ },
+ {
+ "name": "http_response_latency_ms",
+ "label": "HTTP.Response Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "http_session_duration_ms",
+ "label": "HTTP.Session Duration(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "http_action_file_size",
+ "label": "HTTP.Action File Size",
+ "type": "int"
+ },
+ {
+ "name": "doh_url",
+ "label": "DoH.URL",
+ "type": "string"
+ },
+ {
+ "name": "doh_host",
+ "label": "DoH.Host",
+ "type": "string"
+ },
+ {
+ "name": "doh_request_line",
+ "label": "DoH.Request Line",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "doh_response_line",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "label": "DoH.Response Line",
+ "type": "string"
+ },
+ {
+ "name": "doh_cookie",
+ "label": "DoH.Cookie",
+ "type": "string"
+ },
+ {
+ "name": "doh_referer",
+ "label": "DoH.Referer",
+ "type": "string"
+ },
+ {
+ "name": "doh_user_agent",
+ "label": "DoH.User Agent",
+ "type": "string"
+ },
+ {
+ "name": "doh_content_length",
+ "label": "DoH.Content Length",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "doh_content_type",
+ "label": "DoH.Content Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "doh_set_cookie",
+ "label": "DoH.Set Cookie",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "doh_version",
+ "label": "DoH.Version",
+ "type": "string"
+ },
+ {
+ "name": "doh_message_id",
+ "label": "DoH.Message ID",
+ "type": "int"
+ },
+ {
+ "name": "doh_qr",
+ "label": "DoH.QR",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "QUERY"
+ },
+ {
+ "code": "1",
+ "value": "REESPONSE"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "doh_opcode",
+ "label": "DoH.OPCODE",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "QUERY"
+ },
+ {
+ "code": "1",
+ "value": "IQUERY"
+ },
+ {
+ "code": "2",
+ "value": "STATUS"
+ },
+ {
+ "code": "5",
+ "value": "UPDATE"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "doh_aa",
+ "label": "DoH.AA",
+ "type": "int"
+ },
+ {
+ "name": "doh_tc",
+ "label": "DoH.TC",
+ "type": "int"
+ },
+ {
+ "name": "doh_rd",
+ "label": "DoH.RD",
+ "type": "int"
+ },
+ {
+ "name": "doh_ra",
+ "label": "DoH.RA",
+ "type": "int"
+ },
+ {
+ "name": "doh_rcode",
+ "label": "DoH.RCODE",
+ "type": "int"
+ },
+ {
+ "name": "doh_qdcount",
+ "label": "DoH.QDCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "doh_ancount",
+ "label": "DoH.ANCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "doh_nscount",
+ "label": "DoH.NSCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "doh_arcount",
+ "label": "DoH.ARCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "doh_qname",
+ "label": "DoH.QNAME",
+ "type": "string"
+ },
+ {
+ "name": "doh_qtype",
+ "label": "DoH.QTYPE",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "A"
+ },
+ {
+ "code": "2",
+ "value": "NS"
+ },
+ {
+ "code": "5",
+ "value": "CNAME"
+ },
+ {
+ "code": "6",
+ "value": "SOA"
+ },
+ {
+ "code": "11",
+ "value": "WKS"
+ },
+ {
+ "code": "12",
+ "value": "PTR"
+ },
+ {
+ "code": "13",
+ "value": "HINFO"
+ },
+ {
+ "code": "11",
+ "value": "WKS"
+ },
+ {
+ "code": "15",
+ "value": "MX"
+ },
+ {
+ "code": "28",
+ "value": "AAAA"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "doh_qclass",
+ "label": "DoH.QCLASS",
+ "type": "int"
+ },
+ {
+ "name": "doh_cname",
+ "label": "DoH.CNAME",
+ "type": "string"
+ },
+ {
+ "name": "doh_sub",
+ "label": "DoH.SUB",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "DNS"
+ },
+ {
+ "code": "2",
+ "value": "DNSSEC"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "doh_rr",
+ "label": "DoH.RR",
+ "type": "string"
+ }
+ ]
+} \ No newline at end of file
diff --git a/log-stream/schema/security_event.json b/log-stream/schema/security_event.json
new file mode 100644
index 0000000..1f6fc79
--- /dev/null
+++ b/log-stream/schema/security_event.json
@@ -0,0 +1,2477 @@
+{
+ "type": "record",
+ "name": "security_event",
+ "namespace": "tsg_galaxy_v3",
+ "doc": {
+ "primary_key": "common_log_id",
+ "partition_key": "common_recv_time",
+ "functions": {
+ "$ref": "public_schema_info.json#/functions"
+ },
+ "schema_query": {
+ "dimensions": [
+ "common_server_ip",
+ "common_client_ip",
+ "common_internal_ip",
+ "common_external_ip",
+ "common_policy_id",
+ "common_action",
+ "common_sled_ip",
+ "common_device_id",
+ "common_client_location",
+ "common_server_location",
+ "common_subscriber_id",
+ "common_client_port",
+ "common_server_port",
+ "common_schema_type",
+ "common_l4_protocol",
+ "common_l7_protocol",
+ "common_data_center",
+ "common_device_group",
+ "common_app_behavior",
+ "common_client_asn",
+ "common_server_asn",
+ "common_start_time",
+ "common_end_time",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number",
+ "common_app_label",
+ "http_host",
+ "http_domain",
+ "http_url",
+ "http_cookie",
+ "http_referer",
+ "http_user_agent",
+ "ssl_sni",
+ "ssl_ja3_hash",
+ "ssl_passthrough_reason",
+ "ssl_client_side_version",
+ "ssl_server_side_version",
+ "ssl_cert_issuer",
+ "ssl_cert_subject",
+ "mail_account",
+ "mail_from",
+ "mail_to",
+ "quic_sni",
+ "quic_version"
+ ],
+ "metrics": [
+ "common_server_ip",
+ "common_client_ip",
+ "common_internal_ip",
+ "common_external_ip",
+ "common_subscriber_id",
+ "common_sled_ip",
+ "common_device_id",
+ "common_sessions",
+ "common_c2s_pkt_num",
+ "common_s2c_pkt_num",
+ "common_c2s_byte_num",
+ "common_s2c_byte_num",
+ "common_mirrored_pkts",
+ "common_mirrored_bytes",
+ "common_con_duration_ms",
+ "common_establish_latency_ms",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number",
+ "common_app_label",
+ "http_host",
+ "http_domain",
+ "http_url",
+ "http_cookie",
+ "http_referer",
+ "http_user_agent",
+ "ssl_sni",
+ "ssl_ja3_hash",
+ "ssl_passthrough_reason",
+ "ssl_client_side_latency",
+ "ssl_server_side_latency",
+ "ssl_cert_issuer",
+ "ssl_cert_subject",
+ "mail_account",
+ "mail_from",
+ "mail_to",
+ "quic_sni"
+ ],
+ "filters": [
+ "common_policy_id",
+ "common_action",
+ "common_address_type",
+ "common_server_ip",
+ "common_client_ip",
+ "common_internal_ip",
+ "common_external_ip",
+ "common_client_port",
+ "common_server_port",
+ "common_client_location",
+ "common_server_location",
+ "common_subscriber_id",
+ "common_c2s_pkt_num",
+ "common_s2c_pkt_num",
+ "common_c2s_byte_num",
+ "common_s2c_byte_num",
+ "common_mirrored_pkts",
+ "common_mirrored_bytes",
+ "common_l4_protocol",
+ "common_l7_protocol",
+ "common_stream_dir",
+ "common_data_center",
+ "common_device_group",
+ "common_app_behavior",
+ "common_sled_ip",
+ "common_device_id",
+ "common_direction",
+ "common_schema_type",
+ "common_client_asn",
+ "common_server_asn",
+ "common_start_time",
+ "common_end_time",
+ "common_con_duration_ms",
+ "common_establish_latency_ms",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number",
+ "common_app_label",
+ "http_host",
+ "http_domain",
+ "http_url",
+ "http_cookie",
+ "http_referer",
+ "http_user_agent",
+ "http_request_content_type",
+ "http_response_content_type",
+ "ssl_sni",
+ "ssl_ja3_hash",
+ "ssl_pinningst",
+ "ssl_intercept_state",
+ "ssl_passthrough_reason",
+ "ssl_client_side_version",
+ "ssl_server_side_version",
+ "ssl_cert_verify",
+ "ssl_client_side_latency",
+ "ssl_server_side_latency",
+ "ssl_cert_issuer",
+ "ssl_cert_subject",
+ "mail_account",
+ "mail_from",
+ "mail_to",
+ "mail_subject",
+ "quic_sni",
+ "quic_version"
+ ],
+ "references": {
+ "$ref": "public_schema_info.json#/schema_query/references"
+ },
+ "details": {
+ "general": [
+ "common_recv_time",
+ "common_log_id",
+ "common_stream_trace_id",
+ "common_direction",
+ "common_stream_dir",
+ "common_start_time",
+ "common_end_time",
+ "common_con_duration_ms",
+ "common_establish_latency_ms",
+ "common_processing_time",
+ "common_ingestion_time",
+ "common_entrance_id",
+ "common_device_id",
+ "common_egress_link_id",
+ "common_ingress_link_id",
+ "common_isp",
+ "common_data_center",
+ "common_device_group",
+ "common_sled_ip"
+ ],
+ "action": [
+ "common_action",
+ "common_sub_action",
+ "common_policy_id",
+ "common_user_tags",
+ "common_user_region"
+ ],
+ "source": [
+ "common_client_ip",
+ "common_internal_ip",
+ "common_client_port",
+ "common_client_location",
+ "common_client_asn",
+ "common_subscriber_id",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number"
+ ],
+ "destination": [
+ "common_server_ip",
+ "common_external_ip",
+ "common_server_port",
+ "common_server_location",
+ "common_server_asn"
+ ],
+ "application": [
+ "common_app_id",
+ "common_userdefine_app_name",
+ "common_app_label",
+ "common_app_surrogate_id",
+ "common_l7_protocol",
+ "common_protocol_label",
+ "common_service_category",
+ "common_service",
+ "common_l4_protocol",
+ "common_app_behavior"
+ ],
+ "transmission": [
+ "common_sessions",
+ "common_c2s_pkt_num",
+ "common_s2c_pkt_num",
+ "common_c2s_byte_num",
+ "common_s2c_byte_num",
+ "common_c2s_pkt_diff",
+ "common_s2c_pkt_diff",
+ "common_c2s_byte_diff",
+ "common_s2c_byte_diff",
+ "common_c2s_ipfrag_num",
+ "common_s2c_ipfrag_num",
+ "common_c2s_tcp_lostlen",
+ "common_s2c_tcp_lostlen",
+ "common_c2s_tcp_unorder_num",
+ "common_s2c_tcp_unorder_num",
+ "common_c2s_pkt_retrans",
+ "common_s2c_pkt_retrans",
+ "common_c2s_byte_retrans",
+ "common_s2c_byte_retrans",
+ "common_first_ttl",
+ "common_tcp_client_isn",
+ "common_tcp_server_isn",
+ "common_mirrored_pkts",
+ "common_mirrored_bytes"
+ ],
+ "other": [
+ "common_address_type",
+ "common_schema_type",
+ "common_device_tag",
+ "common_encapsulation",
+ "common_tunnels",
+ "common_address_list",
+ "common_has_dup_traffic",
+ "common_stream_error",
+ "common_link_info_c2s",
+ "common_link_info_s2c",
+ "common_packet_capture_file"
+ ]
+ }
+ },
+ "schema_type": {
+ "BASE": {
+ "$ref": "public_schema_info.json#/schema_type/BASE"
+ },
+ "HTTP": {
+ "$ref": "public_schema_info.json#/schema_type/HTTP"
+ },
+ "MAIL": {
+ "$ref": "public_schema_info.json#/schema_type/MAIL"
+ },
+ "DNS": {
+ "$ref": "public_schema_info.json#/schema_type/DNS"
+ },
+ "SSL": {
+ "$ref": "public_schema_info.json#/schema_type/SSL"
+ },
+ "QUIC": {
+ "$ref": "public_schema_info.json#/schema_type/QUIC"
+ },
+ "FTP": {
+ "$ref": "public_schema_info.json#/schema_type/FTP"
+ },
+ "BGP": {
+ "$ref": "public_schema_info.json#/schema_type/BGP"
+ },
+ "SIP": {
+ "$ref": "public_schema_info.json#/schema_type/SIP"
+ },
+ "RTP": {
+ "$ref": "public_schema_info.json#/schema_type/RTP"
+ },
+ "APP": {
+ "$ref": "public_schema_info.json#/schema_type/APP"
+ },
+ "SSH": {
+ "$ref": "public_schema_info.json#/schema_type/SSH"
+ },
+ "Stratum": {
+ "$ref": "public_schema_info.json#/schema_type/Stratum"
+ }
+ },
+ "default_columns": [
+ "common_recv_time",
+ "common_log_id",
+ "common_policy_id",
+ "common_subscriber_id",
+ "common_client_ip",
+ "common_server_ip",
+ "common_server_port",
+ "common_schema_type"
+ ],
+ "internal_columns": [
+ "common_recv_time",
+ "common_log_id",
+ "common_processing_time",
+ "common_ingestion_time",
+ "common_userdefine_app_name",
+ "common_tunnels",
+ "common_packet_capture_file",
+ "http_request_body",
+ "http_response_body",
+ "mail_eml_file",
+ "rtp_pcap_path"
+ ],
+ "tunnel_type": {
+ "$ref": "public_schema_info.json#/tunnel_type"
+ }
+ },
+ "fields": [
+ {
+ "name": "common_recv_time",
+ "label": "Receive Time",
+ "doc": {
+ "constraints": {
+ "type": "timestamp"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_log_id",
+ "label": "Log ID",
+ "doc": {
+ "format": {
+ "functions": "snowflake_id"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_policy_id",
+ "label": "Policy ID",
+ "type": "long"
+ },
+ {
+ "name": "common_subscriber_id",
+ "label": "Subscriber ID",
+ "type": "string"
+ },
+ {
+ "name": "common_imei",
+ "label": "IMEI",
+ "type": "string"
+ },
+ {
+ "name": "common_imsi",
+ "label": "IMSI",
+ "type": "string"
+ },
+ {
+ "name": "common_phone_number",
+ "label": "Phone Number",
+ "type": "string"
+ },
+ {
+ "name": "common_client_ip",
+ "label": "Client IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "geo_asn,radius_match",
+ "appendTo": "common_client_asn,common_subscriber_id"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_internal_ip",
+ "label": "Internal IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "if",
+ "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_client_port",
+ "label": "Client Port",
+ "type": "int"
+ },
+ {
+ "name": "common_l4_protocol",
+ "label": "L4 Protocol",
+ "type": "string"
+ },
+ {
+ "name": "common_address_type",
+ "label": "Address Type",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "4",
+ "value": "ipv4"
+ },
+ {
+ "code": "6",
+ "value": "ipv6"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_server_ip",
+ "label": "Server IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "geo_asn",
+ "appendTo": "common_server_asn"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_server_port",
+ "label": "Server Port",
+ "type": "int"
+ },
+ {
+ "name": "common_external_ip",
+ "label": "External IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "if",
+ "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_action",
+ "label": "Action",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "Monitor"
+ },
+ {
+ "code": "2",
+ "value": "Intercept"
+ },
+ {
+ "code": "16",
+ "value": "Deny"
+ },
+ {
+ "code": "128",
+ "value": "Allow"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_direction",
+ "label": "Direction",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "69",
+ "value": "outbound"
+ },
+ {
+ "code": "73",
+ "value": "inbound"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_entrance_id",
+ "label": "Entrance ID",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_sled_ip",
+ "label": "Sled IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_client_location",
+ "label": "Client Location",
+ "type": "string"
+ },
+ {
+ "name": "common_client_asn",
+ "label": "Client ASN",
+ "type": "string"
+ },
+ {
+ "name": "common_server_location",
+ "label": "Server Location",
+ "type": "string"
+ },
+ {
+ "name": "common_server_asn",
+ "label": "Server ASN",
+ "type": "string"
+ },
+ {
+ "name": "common_sessions",
+ "label": "Sessions",
+ "doc": {
+ "visibility": "disabled",
+ "format": {
+ "functions": "set_value",
+ "param": "1"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_pkt_num",
+ "label": "Packets Sent",
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_pkt_num",
+ "label": "Packets Received",
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_byte_num",
+ "label": "Bytes Sent",
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_byte_num",
+ "label": "Bytes Received",
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_pkt_diff",
+ "label": "Packets Sent(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_pkt_diff",
+ "label": "Packets Received(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_byte_diff",
+ "label": "Bytes Sent(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_byte_diff",
+ "label": "Bytes Received(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_service",
+ "label": "Service",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_schema_type",
+ "label": "Schema Type",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "BASE",
+ "value": "BASE"
+ },
+ {
+ "code": "HTTP",
+ "value": "HTTP"
+ },
+ {
+ "code": "MAIL",
+ "value": "MAIL"
+ },
+ {
+ "code": "DNS",
+ "value": "DNS"
+ },
+ {
+ "code": "SSL",
+ "value": "SSL"
+ },
+ {
+ "code": "QUIC",
+ "value": "QUIC"
+ },
+ {
+ "code": "FTP",
+ "value": "FTP"
+ },
+ {
+ "code": "SIP",
+ "value": "SIP"
+ },
+ {
+ "code": "RTP",
+ "value": "RTP"
+ },
+ {
+ "code": "SSH",
+ "value": "SSH"
+ },
+ {
+ "code": "Stratum",
+ "value": "Stratum"
+ }
+ ]
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_user_tags",
+ "label": "User Tags",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_sub_action",
+ "label": "Sub Action",
+ "doc": {
+ "data": [
+ {
+ "code": "allow",
+ "value": "Allow"
+ },
+ {
+ "code": "deny",
+ "value": "Deny"
+ },
+ {
+ "code": "monitor",
+ "value": "Monitor"
+ },
+ {
+ "code": "replace",
+ "value": "Replace"
+ },
+ {
+ "code": "redirect",
+ "value": "Redirect"
+ },
+ {
+ "code": "insert",
+ "value": "Insert"
+ },
+ {
+ "code": "hijack",
+ "value": "Hijack"
+ }
+ ],
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_user_region",
+ "label": "User Region",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_device_id",
+ "label": "Device ID",
+ "type": "string"
+ },
+ {
+ "name": "common_egress_link_id",
+ "label": "Egress Link ID",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_ingress_link_id",
+ "label": "Ingress Link ID",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_isp",
+ "label": "ISP",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_device_tag",
+ "label": "Device Tag",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_data_center",
+ "label": "Data Center",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "device_tag.json#",
+ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
+ "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_device_group",
+ "label": "Device Group",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "device_tag.json#",
+ "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']",
+ "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_app_behavior",
+ "label": "Application Behavior",
+ "type": "string"
+ },
+ {
+ "name": "common_encapsulation",
+ "label": "Encapsulation",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "public_schema_info.json#/fields/common_encapsulation/data"
+ },
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_app_label",
+ "label": "Application Label",
+ "type": "string"
+ },
+ {
+ "name": "common_tunnels",
+ "label": "Tunnels",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_protocol_label",
+ "label": "Protocol Label",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_app_id",
+ "label": "Application ID",
+ "type": "string",
+ "doc": {
+ "visibility": "disabled"
+ }
+ },
+ {
+ "name": "common_userdefine_app_name",
+ "label": "User Define APP Name",
+ "type": "string"
+ },
+ {
+ "name": "common_app_surrogate_id",
+ "label": "Surrogate ID",
+ "type": "string"
+ },
+ {
+ "name": "common_l7_protocol",
+ "label": "L7 Protocol",
+ "type": "string"
+ },
+ {
+ "name": "common_service_category",
+ "label": "FQDN Category",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "has"
+ },
+ "dict_location": {
+ "path": "/v1/category/dict",
+ "key": "categoryId",
+ "value": "categoryName"
+ }
+ },
+ "type": {
+ "type": "array",
+ "items": "int"
+ }
+ },
+ {
+ "name": "common_start_time",
+ "label": "Start Time",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "timestamp"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_end_time",
+ "label": "End Time",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "timestamp"
+ },
+ "format": {
+ "functions": "get_value",
+ "appendTo": "common_recv_time"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_establish_latency_ms",
+ "label": "Establish Latency(ms)",
+ "type": "long"
+ },
+ {
+ "name": "common_con_duration_ms",
+ "label": "Duration(ms)",
+ "type": "long"
+ },
+ {
+ "name": "common_stream_dir",
+ "label": "Stream Direction",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "c2s"
+ },
+ {
+ "code": "2",
+ "value": "s2c"
+ },
+ {
+ "code": "3",
+ "value": "double"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_address_list",
+ "label": "Address List",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_has_dup_traffic",
+ "label": "Duplication Traffic",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data"
+ },
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_stream_error",
+ "label": "Stream Error",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_stream_trace_id",
+ "label": "Session ID",
+ "type": "long"
+ },
+ {
+ "name": "common_link_info_c2s",
+ "label": "Link Info(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_link_info_s2c",
+ "label": "Link Info(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_packet_capture_file",
+ "label": "Packet Capture File",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_c2s_ipfrag_num",
+ "label": "Fragmentation Packets(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_ipfrag_num",
+ "label": "Fragmentation Packets(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_tcp_lostlen",
+ "label": "Sequence Gap Loss(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_tcp_lostlen",
+ "label": "Sequence Gap Loss(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_tcp_unorder_num",
+ "label": "Unorder Packets(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_tcp_unorder_num",
+ "label": "Unorder Packets(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_pkt_retrans",
+ "label": "Packet Retransmission(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_pkt_retrans",
+ "label": "Packet Retransmission(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_byte_retrans",
+ "label": "Byte Retransmission(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_byte_retrans",
+ "label": "Byte Retransmission(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_tcp_client_isn",
+ "label": "TCP Client ISN",
+ "type": "long"
+ },
+ {
+ "name": "common_tcp_server_isn",
+ "label": "TCP Server ISN",
+ "type": "long"
+ },
+ {
+ "name": "common_first_ttl",
+ "label": "First TTL",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_processing_time",
+ "label": "Processing Time",
+ "doc": {
+ "constraints": {
+ "type": "timestamp"
+ },
+ "format": {
+ "functions": "current_timestamp"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_ingestion_time",
+ "label": "Ingestion Time",
+ "doc": {
+ "constraints": {
+ "type": "timestamp"
+ },
+ "format": {
+ "functions": "ingestion_time"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_mirrored_pkts",
+ "label": "Mirrored Packets",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_mirrored_bytes",
+ "label": "Mirrored Bytes",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "http_url",
+ "label": "HTTP.URL",
+ "type": "string"
+ },
+ {
+ "name": "http_host",
+ "label": "HTTP.Host",
+ "doc": {
+ "format": {
+ "functions": "sub_domain",
+ "appendTo": "http_domain"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_domain",
+ "label": "HTTP.Domain",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_line",
+ "label": "HTTP.Request Line",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_line",
+ "label": "HTTP.Response Line",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_header",
+ "label": "HTTP.Request Header",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_header",
+ "label": "HTTP.Response Header",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_content",
+ "label": "HTTP.Request Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_content_length",
+ "label": "HTTP.Request Content Length",
+ "type": "string"
+ },
+ {
+ "name": "http_request_content_type",
+ "label": "HTTP.Request Content Type",
+ "type": "string"
+ },
+ {
+ "name": "http_response_content",
+ "label": "HTTP.Response Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_content_length",
+ "label": "HTTP.Response Content Length",
+ "type": "string"
+ },
+ {
+ "name": "http_response_content_type",
+ "label": "HTTP.Response Content Type",
+ "type": "string"
+ },
+ {
+ "name": "http_request_body",
+ "label": "HTTP.Request Body",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_body",
+ "label": "HTTP.Response Body",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_body_key",
+ "label": "HTTP.Request Body Key",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_body_key",
+ "label": "HTTP.Response Body Key",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_proxy_flag",
+ "label": "HTTP.Proxy Flag",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "http_sequence",
+ "label": "HTTP.Sequence",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "http_snapshot",
+ "label": "HTTP.Snapshot",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_cookie",
+ "label": "HTTP.Cookie",
+ "type": "string"
+ },
+ {
+ "name": "http_referer",
+ "label": "HTTP.Referer",
+ "type": "string"
+ },
+ {
+ "name": "http_user_agent",
+ "label": "HTTP.User Agent",
+ "type": "string"
+ },
+ {
+ "name": "http_content_length",
+ "label": "HTTP.Content Length",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_content_type",
+ "label": "HTTP.Content Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_set_cookie",
+ "label": "HTTP.Set Cookie",
+ "type": "string"
+ },
+ {
+ "name": "http_version",
+ "label": "HTTP.Version",
+ "type": "string"
+ },
+ {
+ "name": "http_response_latency_ms",
+ "label": "HTTP.Response Latency(ms)",
+ "type": "long"
+ },
+ {
+ "name": "http_action_file_size",
+ "label": "HTTP.Action File Size",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "http_session_duration_ms",
+ "label": "HTTP.Session Duration(ms)",
+ "type": "long"
+ },
+ {
+ "name": "mail_protocol_type",
+ "label": "Mail.Protocol Type",
+ "type": "string"
+ },
+ {
+ "name": "mail_account",
+ "label": "Mail.Account",
+ "type": "string"
+ },
+ {
+ "name": "mail_from_cmd",
+ "label": "Mail.From CMD",
+ "type": "string"
+ },
+ {
+ "name": "mail_to_cmd",
+ "label": "Mail.To CMD",
+ "type": "string"
+ },
+ {
+ "name": "mail_from",
+ "label": "Mail.From",
+ "doc": {
+ "constraints": {
+ "type": "email"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_to",
+ "label": "Mail.To",
+ "doc": {
+ "constraints": {
+ "type": "email"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_cc",
+ "label": "Mail.CC",
+ "type": "string"
+ },
+ {
+ "name": "mail_bcc",
+ "label": "Mail.BCC",
+ "type": "string"
+ },
+ {
+ "name": "mail_subject",
+ "label": "Mail.Subject",
+ "doc": {
+ "format": {
+ "functions": "decode_of_base64",
+ "param": "$.mail_subject_charset"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_subject_charset",
+ "label": "Mail.Subject Charset",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_content",
+ "label": "Mail.Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_content_charset",
+ "label": "Mail.Content Charset",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_attachment_name",
+ "label": "Mail.Attachment",
+ "doc": {
+ "format": {
+ "functions": "decode_of_base64",
+ "param": "$.mail_attachment_name_charset"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_attachment_name_charset",
+ "label": "Mail.Attachment Charset",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_attachment_content",
+ "label": "Mail.Attachment Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_eml_file",
+ "label": "Mail.EML File",
+ "doc": {
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_snapshot",
+ "label": "Mail.Snapshot",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "dns_message_id",
+ "label": "DNS.Message ID",
+ "type": "int"
+ },
+ {
+ "name": "dns_qr",
+ "label": "DNS.QR",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "QUERY"
+ },
+ {
+ "code": "1",
+ "value": "RESPONSE"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "dns_opcode",
+ "label": "DNS.OPCODE",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "QUERY"
+ },
+ {
+ "code": "1",
+ "value": "IQUERY"
+ },
+ {
+ "code": "2",
+ "value": "STATUS"
+ },
+ {
+ "code": "5",
+ "value": "UPDATE"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "dns_aa",
+ "label": "DNS.AA",
+ "type": "int"
+ },
+ {
+ "name": "dns_tc",
+ "label": "DNS.TC",
+ "type": "int"
+ },
+ {
+ "name": "dns_rd",
+ "label": "DNS.RD",
+ "type": "int"
+ },
+ {
+ "name": "dns_ra",
+ "label": "DNS.RA",
+ "type": "int"
+ },
+ {
+ "name": "dns_rcode",
+ "label": "DNS.RCODE",
+ "type": "int"
+ },
+ {
+ "name": "dns_qdcount",
+ "label": "DNS.QDCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "dns_ancount",
+ "label": "DNS.ANCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "dns_nscount",
+ "label": "DNS.NSCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "dns_arcount",
+ "label": "DNS.ARCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "dns_qname",
+ "label": "DNS.QNAME",
+ "type": "string"
+ },
+ {
+ "name": "dns_qtype",
+ "label": "DNS.QTYPE",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "A"
+ },
+ {
+ "code": "2",
+ "value": "NS"
+ },
+ {
+ "code": "5",
+ "value": "CNAME"
+ },
+ {
+ "code": "6",
+ "value": "SOA"
+ },
+ {
+ "code": "11",
+ "value": "WKS"
+ },
+ {
+ "code": "12",
+ "value": "PTR"
+ },
+ {
+ "code": "13",
+ "value": "HINFO"
+ },
+ {
+ "code": "11",
+ "value": "WKS"
+ },
+ {
+ "code": "15",
+ "value": "MX"
+ },
+ {
+ "code": "28",
+ "value": "AAAA"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "dns_qclass",
+ "label": "DNS.QCLASS",
+ "type": "int"
+ },
+ {
+ "name": "dns_cname",
+ "label": "DNS.CNAME",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "dns_sub",
+ "label": "DNS.SUB",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "DNS"
+ },
+ {
+ "code": "2",
+ "value": "DNSSEC"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "dns_rr",
+ "label": "DNS.RR",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "dns_response_latency_ms",
+ "label": "DNS.Response Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_version",
+ "label": "SSL.Version",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_sni",
+ "label": "SSL.SNI",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_san",
+ "label": "SSL.SAN",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_cn",
+ "label": "SSL.CN",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_pinningst",
+ "label": "SSL.Pinning",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "Not Pinning"
+ },
+ {
+ "code": "1",
+ "value": "Pinning"
+ },
+ {
+ "code": "2",
+ "value": "Maybe Pinning"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_intercept_state",
+ "label": "SSL.Intercept State",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "Passthrough"
+ },
+ {
+ "code": "1",
+ "value": "Intercept"
+ },
+ {
+ "code": "2",
+ "value": "Shutdown"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_passthrough_reason",
+ "label": "SSL.Passthrough Reason",
+ "type": "string"
+ },
+ {
+ "name": "ssl_server_side_latency",
+ "label": "SSL.Server Side Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_client_side_latency",
+ "label": "SSL.Client Side Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_server_side_version",
+ "label": "SSL.Server Side Version",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_client_side_version",
+ "label": "SSL.Client Side Version",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_cert_verify",
+ "label": "SSL.Certificate Verify",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "No"
+ },
+ {
+ "code": "1",
+ "value": "Yes"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_error",
+ "label": "SSL.Error",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_con_latency_ms",
+ "label": "SSL.Connection Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_ja3_fingerprint",
+ "label": "SSL.JA3",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_ja3_hash",
+ "label": "SSL.JA3 hash",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_cert_issuer",
+ "label": "SSL.Issuer",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "type": "items"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_cert_subject",
+ "label": "SSL.Subject",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "type": "items"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "quic_version",
+ "label": "Quic.Version",
+ "type": "string"
+ },
+ {
+ "name": "quic_sni",
+ "label": "Quic.SNI",
+ "type": "string"
+ },
+ {
+ "name": "quic_user_agent",
+ "label": "Quic.User Agent",
+ "type": "string"
+ },
+ {
+ "name": "ftp_account",
+ "label": "FTP.Account",
+ "type": "string"
+ },
+ {
+ "name": "ftp_url",
+ "label": "FTP.URL",
+ "type": "string"
+ },
+ {
+ "name": "ftp_content",
+ "label": "FTP.Content",
+ "type": "string"
+ },
+ {
+ "name": "ftp_link_type",
+ "label": "FTP.Link Type",
+ "type": "string"
+ },
+ {
+ "name": "bgp_type",
+ "label": "BGP.Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "bgp_as_num",
+ "label": "BGP.AS Number",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "bgp_route",
+ "label": "BGP.Route",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "voip_calling_account",
+ "label": "VoIP.Calling Account",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "voip_called_account",
+ "label": "VoIP.Called Account",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "voip_calling_number",
+ "label": "VoIP.Calling Number",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "voip_called_number",
+ "label": "VoIP.Called Number",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "streaming_media_url",
+ "label": "Streaming.Media URL",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "streaming_media_protocol",
+ "label": "Streaming.Media Protocol",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "app_extra_info",
+ "label": "APP.Extra Info",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_call_id",
+ "label": "SIP.Call-ID",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_originator_description",
+ "label": "SIP.Originator",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_responder_description",
+ "label": "SIP.Responder",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_user_agent",
+ "label": "SIP.User-Agent",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_server",
+ "label": "SIP.Server",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_originator_sdp_connect_ip",
+ "label": "SIP.Originator IP",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_originator_sdp_media_port",
+ "label": "SIP.Originator Port",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "sip_originator_sdp_media_type",
+ "label": "SIP.Originator Media Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_originator_sdp_content",
+ "label": "SIP.Originator Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_responder_sdp_connect_ip",
+ "label": "SIP.Responder IP",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_responder_sdp_media_port",
+ "label": "SIP.Responder Port",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "sip_responder_sdp_media_type",
+ "label": "SIP.Responder Media Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_responder_sdp_content",
+ "label": "SIP.Responder Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_duration_s",
+ "label": "SIP.Duration(s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "sip_bye",
+ "label": "SIP.Bye",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "rtp_payload_type_c2s",
+ "label": "RTP.Payload Type(c2s)",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "PCMU"
+ },
+ {
+ "code": "1",
+ "value": "1016"
+ },
+ {
+ "code": "2",
+ "value": "G721"
+ },
+ {
+ "code": "3",
+ "value": "GSM"
+ },
+ {
+ "code": "4",
+ "value": "G723"
+ },
+ {
+ "code": "5",
+ "value": "DVI4_8000"
+ },
+ {
+ "code": "6",
+ "value": "DVI4_16000"
+ },
+ {
+ "code": "7",
+ "value": "LPC"
+ },
+ {
+ "code": "8",
+ "value": "PCMA"
+ },
+ {
+ "code": "9",
+ "value": "G722"
+ },
+ {
+ "code": "10",
+ "value": "L16_STEREO"
+ },
+ {
+ "code": "11",
+ "value": "L16_MONO"
+ },
+ {
+ "code": "12",
+ "value": "QCELP"
+ },
+ {
+ "code": "13",
+ "value": "CN"
+ },
+ {
+ "code": "14",
+ "value": "MPA"
+ },
+ {
+ "code": "15",
+ "value": "G728"
+ },
+ {
+ "code": "16",
+ "value": "DVI4_11025"
+ },
+ {
+ "code": "17",
+ "value": "DVI4_22050"
+ },
+ {
+ "code": "18",
+ "value": "G729"
+ },
+ {
+ "code": "19",
+ "value": "CN_OLD"
+ },
+ {
+ "code": "25",
+ "value": "CELB"
+ },
+ {
+ "code": "26",
+ "value": "JPEG"
+ },
+ {
+ "code": "28",
+ "value": "NV"
+ },
+ {
+ "code": "31",
+ "value": "H261"
+ },
+ {
+ "code": "32",
+ "value": "MPV"
+ },
+ {
+ "code": "33",
+ "value": "MP2T"
+ },
+ {
+ "code": "34",
+ "value": "H263"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "rtp_payload_type_s2c",
+ "label": "RTP.Payload Type(s2c)",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "PCMU"
+ },
+ {
+ "code": "1",
+ "value": "1016"
+ },
+ {
+ "code": "2",
+ "value": "G721"
+ },
+ {
+ "code": "3",
+ "value": "GSM"
+ },
+ {
+ "code": "4",
+ "value": "G723"
+ },
+ {
+ "code": "5",
+ "value": "DVI4_8000"
+ },
+ {
+ "code": "6",
+ "value": "DVI4_16000"
+ },
+ {
+ "code": "7",
+ "value": "LPC"
+ },
+ {
+ "code": "8",
+ "value": "PCMA"
+ },
+ {
+ "code": "9",
+ "value": "G722"
+ },
+ {
+ "code": "10",
+ "value": "L16_STEREO"
+ },
+ {
+ "code": "11",
+ "value": "L16_MONO"
+ },
+ {
+ "code": "12",
+ "value": "QCELP"
+ },
+ {
+ "code": "13",
+ "value": "CN"
+ },
+ {
+ "code": "14",
+ "value": "MPA"
+ },
+ {
+ "code": "15",
+ "value": "G728"
+ },
+ {
+ "code": "16",
+ "value": "DVI4_11025"
+ },
+ {
+ "code": "17",
+ "value": "DVI4_22050"
+ },
+ {
+ "code": "18",
+ "value": "G729"
+ },
+ {
+ "code": "19",
+ "value": "CN_OLD"
+ },
+ {
+ "code": "25",
+ "value": "CELB"
+ },
+ {
+ "code": "26",
+ "value": "JPEG"
+ },
+ {
+ "code": "28",
+ "value": "NV"
+ },
+ {
+ "code": "31",
+ "value": "H261"
+ },
+ {
+ "code": "32",
+ "value": "MPV"
+ },
+ {
+ "code": "33",
+ "value": "MP2T"
+ },
+ {
+ "code": "34",
+ "value": "H263"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "rtp_pcap_path",
+ "label": "RTP.PCAP",
+ "doc": {
+ "visibility": "disabled",
+ "allow_query": "false",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "rtp_originator_dir",
+ "label": "RTP.Direction",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "unknown"
+ },
+ {
+ "code": "1",
+ "value": "c2s"
+ },
+ {
+ "code": "2",
+ "value": "s2c"
+ }
+ ],
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssh_version",
+ "label": "SSH.Version",
+ "type": "string"
+ },
+ {
+ "name": "ssh_auth_success",
+ "label": "SSH.Authentication Result",
+ "type": "string"
+ },
+ {
+ "name": "ssh_client_version",
+ "label": "SSH.Client Version",
+ "type": "string"
+ },
+ {
+ "name": "ssh_server_version",
+ "label": "SSH.Server Version",
+ "type": "string"
+ },
+ {
+ "name": "ssh_cipher_alg",
+ "label": "SSH.Encryption Algorithm",
+ "type": "string"
+ },
+ {
+ "name": "ssh_mac_alg",
+ "label": "SSH.Signing Algorithm",
+ "type": "string"
+ },
+ {
+ "name": "ssh_compression_alg",
+ "label": "SSH.Compression Algorithm",
+ "type": "string"
+ },
+ {
+ "name": "ssh_kex_alg",
+ "label": "SSH. Key Exchange Algorithm",
+ "type": "string"
+ },
+ {
+ "name": "ssh_host_key_alg",
+ "label": "SSH.Server Host Key Algorithm",
+ "type": "string"
+ },
+ {
+ "name": "ssh_host_key",
+ "label": "SSH.Server Key Fingerprint",
+ "type": "string"
+ },
+ {
+ "name": "ssh_hassh",
+ "label": "SSH.HASSH",
+ "type": "string"
+ },
+ {
+ "name": "stratum_cryptocurrency",
+ "label": "Stratum.Cryptocurrency",
+ "type": "string"
+ },
+ {
+ "name": "stratum_mining_pools",
+ "label": "Stratum.Mining Pools",
+ "type": "string"
+ },
+ {
+ "name": "stratum_mining_program",
+ "label": "Stratum.Mining Program",
+ "type": "string"
+ }
+ ]
+} \ No newline at end of file
diff --git a/log-stream/schema/session_record.json b/log-stream/schema/session_record.json
new file mode 100644
index 0000000..a71325d
--- /dev/null
+++ b/log-stream/schema/session_record.json
@@ -0,0 +1,2465 @@
+{
+ "type": "record",
+ "name": "session_record",
+ "namespace": "tsg_galaxy_v3",
+ "doc": {
+ "primary_key": "common_log_id",
+ "partition_key": "common_recv_time",
+ "index_table": "session_record_common_client_ip,session_record_common_server_ip,session_record_http_domain",
+ "functions": {
+ "$ref": "public_schema_info.json#/functions"
+ },
+ "schema_query": {
+ "dimensions": [
+ "common_server_ip",
+ "common_client_ip",
+ "common_internal_ip",
+ "common_external_ip",
+ "common_sled_ip",
+ "common_device_id",
+ "common_client_location",
+ "common_server_location",
+ "common_subscriber_id",
+ "common_client_port",
+ "common_server_port",
+ "common_schema_type",
+ "common_l4_protocol",
+ "common_l7_protocol",
+ "common_data_center",
+ "common_device_group",
+ "common_app_behavior",
+ "common_client_asn",
+ "common_server_asn",
+ "common_start_time",
+ "common_end_time",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number",
+ "common_app_label",
+ "http_host",
+ "http_domain",
+ "http_url",
+ "http_cookie",
+ "http_referer",
+ "http_user_agent",
+ "ssl_sni",
+ "ssl_ja3_hash",
+ "ssl_cert_issuer",
+ "ssl_cert_subject",
+ "quic_sni",
+ "quic_version"
+ ],
+ "metrics": [
+ "common_server_ip",
+ "common_client_ip",
+ "common_internal_ip",
+ "common_external_ip",
+ "common_subscriber_id",
+ "common_sled_ip",
+ "common_device_id",
+ "common_c2s_pkt_num",
+ "common_s2c_pkt_num",
+ "common_c2s_byte_num",
+ "common_s2c_byte_num",
+ "common_sessions",
+ "common_con_duration_ms",
+ "common_establish_latency_ms",
+ "common_c2s_ipfrag_num",
+ "common_s2c_ipfrag_num",
+ "common_c2s_tcp_lostlen",
+ "common_s2c_tcp_lostlen",
+ "common_c2s_tcp_unorder_num",
+ "common_s2c_tcp_unorder_num",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number",
+ "common_app_label",
+ "http_host",
+ "http_domain",
+ "http_url",
+ "http_cookie",
+ "http_referer",
+ "http_user_agent",
+ "ssl_sni",
+ "ssl_ja3_hash",
+ "ssl_cert_issuer",
+ "ssl_cert_subject",
+ "quic_sni"
+ ],
+ "filters": [
+ "common_address_type",
+ "common_server_ip",
+ "common_client_ip",
+ "common_internal_ip",
+ "common_external_ip",
+ "common_client_port",
+ "common_server_port",
+ "common_client_location",
+ "common_server_location",
+ "common_subscriber_id",
+ "common_c2s_pkt_num",
+ "common_s2c_pkt_num",
+ "common_c2s_byte_num",
+ "common_s2c_byte_num",
+ "common_c2s_ipfrag_num",
+ "common_s2c_ipfrag_num",
+ "common_c2s_tcp_lostlen",
+ "common_s2c_tcp_lostlen",
+ "common_c2s_tcp_unorder_num",
+ "common_s2c_tcp_unorder_num",
+ "common_l4_protocol",
+ "common_l7_protocol",
+ "common_stream_dir",
+ "common_direction",
+ "common_data_center",
+ "common_device_group",
+ "common_app_behavior",
+ "common_sled_ip",
+ "common_device_id",
+ "common_schema_type",
+ "common_client_asn",
+ "common_server_asn",
+ "common_start_time",
+ "common_end_time",
+ "common_con_duration_ms",
+ "common_establish_latency_ms",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number",
+ "common_app_label",
+ "http_host",
+ "http_domain",
+ "http_url",
+ "http_cookie",
+ "http_referer",
+ "http_user_agent",
+ "ssl_sni",
+ "ssl_ja3_hash",
+ "ssl_cert_issuer",
+ "ssl_cert_subject",
+ "quic_sni",
+ "quic_version"
+ ],
+ "references": {
+ "$ref": "public_schema_info.json#/schema_query/references"
+ },
+ "details": {
+ "general": [
+ "common_recv_time",
+ "common_log_id",
+ "common_stream_trace_id",
+ "common_direction",
+ "common_stream_dir",
+ "common_start_time",
+ "common_end_time",
+ "common_con_duration_ms",
+ "common_establish_latency_ms",
+ "common_processing_time",
+ "common_ingestion_time",
+ "common_entrance_id",
+ "common_device_id",
+ "common_egress_link_id",
+ "common_ingress_link_id",
+ "common_isp",
+ "common_data_center",
+ "common_device_group",
+ "common_sled_ip"
+ ],
+ "source": [
+ "common_client_ip",
+ "common_internal_ip",
+ "common_client_port",
+ "common_client_location",
+ "common_client_asn",
+ "common_subscriber_id",
+ "common_imei",
+ "common_imsi",
+ "common_phone_number"
+ ],
+ "destination": [
+ "common_server_ip",
+ "common_external_ip",
+ "common_server_port",
+ "common_server_location",
+ "common_server_asn"
+ ],
+ "application": [
+ "common_app_id",
+ "common_userdefine_app_name",
+ "common_app_label",
+ "common_app_surrogate_id",
+ "common_l7_protocol",
+ "common_protocol_label",
+ "common_service_category",
+ "common_service",
+ "common_l4_protocol",
+ "common_app_behavior"
+ ],
+ "transmission": [
+ "common_sessions",
+ "common_c2s_pkt_num",
+ "common_s2c_pkt_num",
+ "common_c2s_byte_num",
+ "common_s2c_byte_num",
+ "common_c2s_pkt_diff",
+ "common_s2c_pkt_diff",
+ "common_c2s_byte_diff",
+ "common_s2c_byte_diff",
+ "common_c2s_ipfrag_num",
+ "common_s2c_ipfrag_num",
+ "common_c2s_tcp_lostlen",
+ "common_s2c_tcp_lostlen",
+ "common_c2s_tcp_unorder_num",
+ "common_s2c_tcp_unorder_num",
+ "common_c2s_pkt_retrans",
+ "common_s2c_pkt_retrans",
+ "common_c2s_byte_retrans",
+ "common_s2c_byte_retrans",
+ "common_first_ttl",
+ "common_tcp_client_isn",
+ "common_tcp_server_isn",
+ "common_mirrored_pkts",
+ "common_mirrored_bytes"
+ ],
+ "other": [
+ "common_address_type",
+ "common_schema_type",
+ "common_device_tag",
+ "common_encapsulation",
+ "common_tunnels",
+ "common_address_list",
+ "common_has_dup_traffic",
+ "common_stream_error",
+ "common_link_info_c2s",
+ "common_link_info_s2c",
+ "common_packet_capture_file",
+ "common_action",
+ "common_sub_action",
+ "common_policy_id",
+ "common_user_tags",
+ "common_user_region"
+ ]
+ }
+ },
+ "schema_type": {
+ "BASE": {
+ "$ref": "public_schema_info.json#/schema_type/BASE"
+ },
+ "HTTP": {
+ "$ref": "public_schema_info.json#/schema_type/HTTP"
+ },
+ "MAIL": {
+ "$ref": "public_schema_info.json#/schema_type/MAIL"
+ },
+ "DNS": {
+ "$ref": "public_schema_info.json#/schema_type/DNS"
+ },
+ "SSL": {
+ "$ref": "public_schema_info.json#/schema_type/SSL"
+ },
+ "QUIC": {
+ "$ref": "public_schema_info.json#/schema_type/QUIC"
+ },
+ "FTP": {
+ "$ref": "public_schema_info.json#/schema_type/FTP"
+ },
+ "BGP": {
+ "$ref": "public_schema_info.json#/schema_type/BGP"
+ },
+ "APP": {
+ "$ref": "public_schema_info.json#/schema_type/APP"
+ },
+ "SSH": {
+ "$ref": "public_schema_info.json#/schema_type/SSH"
+ },
+ "Stratum": {
+ "$ref": "public_schema_info.json#/schema_type/Stratum"
+ }
+ },
+ "default_columns": [
+ "common_recv_time",
+ "common_log_id",
+ "common_subscriber_id",
+ "common_client_ip",
+ "common_server_ip",
+ "common_server_port",
+ "common_schema_type"
+ ],
+ "internal_columns": [
+ "common_recv_time",
+ "common_log_id",
+ "common_processing_time",
+ "common_ingestion_time",
+ "common_userdefine_app_name",
+ "common_tunnels",
+ "common_packet_capture_file",
+ "rtp_pcap_path",
+ "http_request_body",
+ "http_response_body",
+ "mail_eml_file"
+ ],
+ "tunnel_type": {
+ "$ref": "public_schema_info.json#/tunnel_type"
+ }
+ },
+ "fields": [
+ {
+ "name": "common_recv_time",
+ "label": "Receive Time",
+ "doc": {
+ "constraints": {
+ "type": "timestamp"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_log_id",
+ "label": "Log ID",
+ "doc": {
+ "format": {
+ "functions": "snowflake_id"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_policy_id",
+ "label": "Policy ID",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_subscriber_id",
+ "label": "Subscriber ID",
+ "type": "string"
+ },
+ {
+ "name": "common_imei",
+ "label": "IMEI",
+ "type": "string"
+ },
+ {
+ "name": "common_imsi",
+ "label": "IMSI",
+ "type": "string"
+ },
+ {
+ "name": "common_phone_number",
+ "label": "Phone Number",
+ "type": "string"
+ },
+ {
+ "name": "common_client_ip",
+ "label": "Client IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "geo_asn,radius_match",
+ "appendTo": "common_client_asn,common_subscriber_id"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_internal_ip",
+ "label": "Internal IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "if",
+ "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_client_port",
+ "label": "Client Port",
+ "type": "int"
+ },
+ {
+ "name": "common_l4_protocol",
+ "label": "L4 Protocol",
+ "type": "string"
+ },
+ {
+ "name": "common_address_type",
+ "label": "Address Type",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "4",
+ "value": "ipv4"
+ },
+ {
+ "code": "6",
+ "value": "ipv6"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_server_ip",
+ "label": "Server IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "geo_asn",
+ "appendTo": "common_server_asn"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_server_port",
+ "label": "Server Port",
+ "type": "int"
+ },
+ {
+ "name": "common_external_ip",
+ "label": "External IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ },
+ "format": {
+ "functions": "if",
+ "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_action",
+ "label": "Action",
+ "doc": {
+ "visibility": "hidden",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "None"
+ },
+ {
+ "code": "1",
+ "value": "Monitor"
+ },
+ {
+ "code": "2",
+ "value": "Intercept"
+ },
+ {
+ "code": "16",
+ "value": "Deny"
+ },
+ {
+ "code": "128",
+ "value": "Allow"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_direction",
+ "label": "Direction",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "69",
+ "value": "outbound"
+ },
+ {
+ "code": "73",
+ "value": "inbound"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_entrance_id",
+ "label": "Entrance ID",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_sled_ip",
+ "label": "Sled IP",
+ "doc": {
+ "constraints": {
+ "type": "ip"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_client_location",
+ "label": "Client Location",
+ "type": "string"
+ },
+ {
+ "name": "common_client_asn",
+ "label": "Client ASN",
+ "type": "string"
+ },
+ {
+ "name": "common_server_location",
+ "label": "Server Location",
+ "type": "string"
+ },
+ {
+ "name": "common_server_asn",
+ "label": "Server ASN",
+ "type": "string"
+ },
+ {
+ "name": "common_sessions",
+ "label": "Sessions",
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_pkt_num",
+ "label": "Packets Sent",
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_pkt_num",
+ "label": "Packets Received",
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_byte_num",
+ "label": "Bytes Sent",
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_byte_num",
+ "label": "Bytes Received",
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_pkt_diff",
+ "label": "Packets Sent(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_pkt_diff",
+ "label": "Packets Received(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_byte_diff",
+ "label": "Bytes Sent(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_byte_diff",
+ "label": "Bytes Received(Diff)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_service",
+ "label": "Service",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_schema_type",
+ "label": "Schema Type",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "BASE",
+ "value": "BASE"
+ },
+ {
+ "code": "MAIL",
+ "value": "MAIL"
+ },
+ {
+ "code": "DNS",
+ "value": "DNS"
+ },
+ {
+ "code": "HTTP",
+ "value": "HTTP"
+ },
+ {
+ "code": "SSL",
+ "value": "SSL"
+ },
+ {
+ "code": "QUIC",
+ "value": "QUIC"
+ },
+ {
+ "code": "FTP",
+ "value": "FTP"
+ },
+ {
+ "code": "SSH",
+ "value": "SSH"
+ },
+ {
+ "code": "Stratum",
+ "value": "Stratum"
+ }
+ ]
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_user_tags",
+ "label": "User Tags",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_sub_action",
+ "label": "Sub Action",
+ "doc": {
+ "data": [
+ {
+ "code": "allow",
+ "value": "Allow"
+ },
+ {
+ "code": "deny",
+ "value": "Deny"
+ },
+ {
+ "code": "monitor",
+ "value": "Monitor"
+ },
+ {
+ "code": "replace",
+ "value": "Replace"
+ },
+ {
+ "code": "redirect",
+ "value": "Redirect"
+ },
+ {
+ "code": "insert",
+ "value": "Insert"
+ },
+ {
+ "code": "hijack",
+ "value": "Hijack"
+ }
+ ],
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_user_region",
+ "label": "User Region",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_device_id",
+ "label": "Device ID",
+ "type": "string"
+ },
+ {
+ "name": "common_egress_link_id",
+ "label": "Egress Link ID",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_ingress_link_id",
+ "label": "Ingress Link ID",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_isp",
+ "label": "ISP",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_device_tag",
+ "label": "Device Tag",
+ "doc": {
+ "visibility": "disabled",
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_data_center",
+ "label": "Data Center",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "device_tag.json#",
+ "key": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagValue']",
+ "value": "$[?(@.tagType=='data_center')].subTags.[?(@.tagType=='data_center')]['tagName']"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_device_group",
+ "label": "Device Group",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "device_tag.json#",
+ "key": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagValue']",
+ "value": "$[?(@.tagType=='device_group')].subTags.[?(@.tagType=='device_group')]['tagName']"
+ },
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_app_behavior",
+ "label": "Application Behavior",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_encapsulation",
+ "label": "Encapsulation",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "public_schema_info.json#/fields/common_encapsulation/data"
+ },
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_app_label",
+ "label": "Application Label",
+ "type": "string"
+ },
+ {
+ "name": "common_tunnels",
+ "label": "Tunnels",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_protocol_label",
+ "label": "Protocol Label",
+ "type": "string"
+ },
+ {
+ "name": "common_app_id",
+ "label": "Application ID",
+ "type": "string",
+ "doc": {
+ "visibility": "hidden"
+ }
+ },
+ {
+ "name": "common_userdefine_app_name",
+ "label": "User Define APP Name",
+ "type": "string"
+ },
+ {
+ "name": "common_app_surrogate_id",
+ "label": "Surrogate ID",
+ "type": "string"
+ },
+ {
+ "name": "common_l7_protocol",
+ "label": "L7 Protocol",
+ "type": "string"
+ },
+ {
+ "name": "common_service_category",
+ "label": "FQDN Category",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+
+ "operator_functions": "has"
+ },
+ "dict_location": {
+ "path": "/v1/category/dict",
+ "key": "categoryId",
+ "value": "categoryName"
+ }
+ },
+ "type": {
+ "type": "array",
+ "items": "int"
+ }
+ },
+ {
+ "name": "common_start_time",
+ "label": "Start Time",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "timestamp"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_end_time",
+ "label": "End Time",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "timestamp"
+ },
+ "format": {
+ "functions": "get_value",
+ "appendTo": "common_recv_time"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_establish_latency_ms",
+ "label": "Establish Latency(ms)",
+ "type": "long"
+ },
+ {
+ "name": "common_con_duration_ms",
+ "label": "Duration(ms)",
+ "type": "long"
+ },
+ {
+ "name": "common_stream_dir",
+ "label": "Stream Direction",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "c2s"
+ },
+ {
+ "code": "2",
+ "value": "s2c"
+ },
+ {
+ "code": "3",
+ "value": "double"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_address_list",
+ "label": "Address List",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_has_dup_traffic",
+ "label": "Duplication Traffic",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": {
+ "$ref": "public_schema_info.json#/fields/common_has_dup_traffic/data"
+ },
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_stream_error",
+ "label": "Stream Error",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_stream_trace_id",
+ "label": "Session ID",
+ "type": "long"
+ },
+ {
+ "name": "common_link_info_c2s",
+ "label": "Link Info(c2s)",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_link_info_s2c",
+ "label": "Link Info(s2c)",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_packet_capture_file",
+ "label": "Packet Capture File",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "common_c2s_ipfrag_num",
+ "label": "Fragmentation Packets(c2s)",
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_ipfrag_num",
+ "label": "Fragmentation Packets(s2c)",
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_tcp_lostlen",
+ "label": "Sequence Gap Loss(c2s)",
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_tcp_lostlen",
+ "label": "Sequence Gap Loss(s2c)",
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_tcp_unorder_num",
+ "label": "Unorder Packets(c2s)",
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_tcp_unorder_num",
+ "label": "Unorder Packets(s2c)",
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_pkt_retrans",
+ "label": "Packet Retransmission(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_pkt_retrans",
+ "label": "Packet Retransmission(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_c2s_byte_retrans",
+ "label": "Byte Retransmission(c2s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_s2c_byte_retrans",
+ "label": "Byte Retransmission(s2c)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_tcp_client_isn",
+ "label": "TCP Client ISN",
+ "type": "long"
+ },
+ {
+ "name": "common_tcp_server_isn",
+ "label": "TCP Server ISN",
+ "type": "long"
+ },
+ {
+ "name": "common_first_ttl",
+ "label": "First TTL",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "int"
+ },
+ {
+ "name": "common_processing_time",
+ "label": "Processing Time",
+ "doc": {
+ "constraints": {
+ "type": "timestamp"
+ },
+ "format": {
+ "functions": "current_timestamp"
+ }
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_ingestion_time",
+ "label": "Ingestion Time",
+ "doc": {
+ "constraints": {
+ "type": "timestamp"
+ },
+ "format": {
+ "functions": "ingestion_time"
+ },
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "common_mirrored_pkts",
+ "label": "Mirrored Packets",
+ "type": "long",
+ "doc": {
+ "visibility": "hidden"
+ }
+ },
+ {
+ "name": "common_mirrored_bytes",
+ "label": "Mirrored Bytes",
+ "type": "long",
+ "doc": {
+ "visibility": "hidden"
+ }
+ },
+ {
+ "name": "http_url",
+ "label": "HTTP.URL",
+ "type": "string"
+ },
+ {
+ "name": "http_host",
+ "label": "HTTP.Host",
+ "type": "string"
+ },
+ {
+ "name": "http_domain",
+ "label": "HTTP.Domain",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_line",
+ "label": "HTTP.Request Line",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_line",
+ "label": "HTTP.Response Line",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_header",
+ "label": "HTTP.Request Headers",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_header",
+ "label": "HTTP.Response Headers",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_content",
+ "label": "HTTP.Request Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_content_length",
+ "label": "HTTP.Request Content Length",
+ "type": "string"
+ },
+ {
+ "name": "http_request_content_type",
+ "label": "HTTP.Request Content Type",
+ "type": "string"
+ },
+ {
+ "name": "http_response_content",
+ "label": "HTTP.Response Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_content_length",
+ "label": "HTTP.Response Content Length",
+ "type": "string"
+ },
+ {
+ "name": "http_response_content_type",
+ "label": "HTTP.Response Content Type",
+ "type": "string"
+ },
+ {
+ "name": "http_request_body",
+ "label": "HTTP.Request Body",
+ "doc": {
+ "allow_query": "false",
+ "visibility": "disabled",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_body",
+ "label": "HTTP.Response Body",
+ "doc": {
+ "allow_query": "false",
+ "visibility": "disabled",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_request_body_key",
+ "label": "HTTP.Request Body Key",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_response_body_key",
+ "label": "HTTP.Response Body Key",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_proxy_flag",
+ "label": "HTTP.Proxy Flag",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "int"
+ },
+ {
+ "name": "http_sequence",
+ "label": "HTTP.Sequence",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "int"
+ },
+ {
+ "name": "http_snapshot",
+ "label": "HTTP.Snapshot",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_cookie",
+ "label": "HTTP.Cookie",
+ "type": "string"
+ },
+ {
+ "name": "http_referer",
+ "label": "HTTP.Referer",
+ "type": "string"
+ },
+ {
+ "name": "http_user_agent",
+ "label": "HTTP.User Agent",
+ "type": "string"
+ },
+ {
+ "name": "http_content_length",
+ "label": "HTTP.Content Length",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_content_type",
+ "label": "HTTP.Content Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "http_set_cookie",
+ "label": "HTTP.Set Cookie",
+ "type": "string"
+ },
+ {
+ "name": "http_version",
+ "label": "HTTP.Version",
+ "type": "string"
+ },
+ {
+ "name": "http_response_latency_ms",
+ "label": "HTTP.Response Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "http_session_duration_ms",
+ "label": "HTTP.Session Duration(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "long"
+ },
+ {
+ "name": "http_action_file_size",
+ "label": "HTTP.Action File Size",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "mail_protocol_type",
+ "label": "Mail.Protocol Type",
+ "type": "string"
+ },
+ {
+ "name": "mail_account",
+ "label": "Mail.Account",
+ "type": "string"
+ },
+ {
+ "name": "mail_from_cmd",
+ "label": "Mail.From CMD",
+ "type": "string"
+ },
+ {
+ "name": "mail_to_cmd",
+ "label": "Mail.To CMD",
+ "type": "string"
+ },
+ {
+ "name": "mail_from",
+ "label": "Mail.From",
+ "doc": {
+ "constraints": {
+ "type": "email"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_to",
+ "label": "Mail.To",
+ "doc": {
+ "constraints": {
+ "type": "email"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_cc",
+ "label": "Mail.CC",
+ "type": "string"
+ },
+ {
+ "name": "mail_bcc",
+ "label": "Mail.BCC",
+ "type": "string"
+ },
+ {
+ "name": "mail_subject",
+ "label": "Mail.Subject",
+ "doc": {
+ "format": {
+ "functions": "decode_of_base64",
+ "param": "$.mail_subject_charset"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_subject_charset",
+ "label": "Mail.Subject Charset",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_content",
+ "label": "Mail.Content",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_content_charset",
+ "label": "Mail.Content Charset",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_attachment_name",
+ "label": "Mail.Attachment",
+ "doc": {
+ "format": {
+ "functions": "decode_of_base64",
+ "param": "$.mail_attachment_name_charset"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_attachment_name_charset",
+ "label": "Mail.Attachment Charset",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_attachment_content",
+ "label": "Mail.Attachment Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_eml_file",
+ "label": "Mail.EML File",
+ "doc": {
+ "allow_query": "false",
+ "constraints": {
+ "type": "file"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "mail_snapshot",
+ "label": "Mail.Snapshot",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "dns_message_id",
+ "label": "DNS.Message ID",
+ "type": "int"
+ },
+ {
+ "name": "dns_qr",
+ "label": "DNS.QR",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "QUERY"
+ },
+ {
+ "code": "1",
+ "value": "RESPONSE"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "dns_opcode",
+ "label": "DNS.OPCODE",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "QUERY"
+ },
+ {
+ "code": "1",
+ "value": "IQUERY"
+ },
+ {
+ "code": "2",
+ "value": "STATUS"
+ },
+ {
+ "code": "5",
+ "value": "UPDATE"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "dns_aa",
+ "label": "DNS.AA",
+ "type": "int"
+ },
+ {
+ "name": "dns_tc",
+ "label": "DNS.TC",
+ "type": "int"
+ },
+ {
+ "name": "dns_rd",
+ "label": "DNS.RD",
+ "type": "int"
+ },
+ {
+ "name": "dns_ra",
+ "label": "DNS.RA",
+ "type": "int"
+ },
+ {
+ "name": "dns_rcode",
+ "label": "DNS.RCODE",
+ "type": "int"
+ },
+ {
+ "name": "dns_qdcount",
+ "label": "DNS.QDCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "dns_ancount",
+ "label": "DNS.ANCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "dns_nscount",
+ "label": "DNS.NSCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "dns_arcount",
+ "label": "DNS.ARCOUNT",
+ "type": "int"
+ },
+ {
+ "name": "dns_qname",
+ "label": "DNS.QNAME",
+ "type": "string"
+ },
+ {
+ "name": "dns_qtype",
+ "label": "DNS.QTYPE",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "A"
+ },
+ {
+ "code": "2",
+ "value": "NS"
+ },
+ {
+ "code": "5",
+ "value": "CNAME"
+ },
+ {
+ "code": "6",
+ "value": "SOA"
+ },
+ {
+ "code": "11",
+ "value": "WKS"
+ },
+ {
+ "code": "12",
+ "value": "PTR"
+ },
+ {
+ "code": "13",
+ "value": "HINFO"
+ },
+ {
+ "code": "11",
+ "value": "WKS"
+ },
+ {
+ "code": "15",
+ "value": "MX"
+ },
+ {
+ "code": "28",
+ "value": "AAAA"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "dns_qclass",
+ "label": "DNS.QCLASS",
+ "type": "int"
+ },
+ {
+ "name": "dns_cname",
+ "label": "DNS.CNAME",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "dns_sub",
+ "label": "DNS.SUB",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "1",
+ "value": "DNS"
+ },
+ {
+ "code": "2",
+ "value": "DNSSEC"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "dns_rr",
+ "label": "DNS.RR",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "dns_response_latency_ms",
+ "label": "DNS.Response Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_version",
+ "label": "SSL.Version",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_sni",
+ "label": "SSL.SNI",
+ "doc": {
+ "format": {
+ "functions": "sub_domain",
+ "appendTo": "http_domain"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_san",
+ "label": "SSL.SAN",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_cn",
+ "label": "SSL.CN",
+ "type": "string"
+ },
+ {
+ "name": "ssl_pinningst",
+ "label": "SSL.Pinning",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "Not Pinning"
+ },
+ {
+ "code": "1",
+ "value": "Pinning"
+ },
+ {
+ "code": "2",
+ "value": "Maybe Pinning"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_intercept_state",
+ "label": "SSL.Intercept State",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "Passthrough"
+ },
+ {
+ "code": "1",
+ "value": "Intercept"
+ },
+ {
+ "code": "2",
+ "value": "Shutdown"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_passthrough_reason",
+ "label": "SSL.Passthrough Reason",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_server_side_latency",
+ "label": "SSL.Server Side Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_client_side_latency",
+ "label": "SSL.Client Side Latency(ms)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_server_side_version",
+ "label": "SSL.Server Side Version",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_client_side_version",
+ "label": "SSL.Client Side Version",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_cert_verify",
+ "label": "SSL.Certificate Verify",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "No"
+ },
+ {
+ "code": "1",
+ "value": "Yes"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssl_error",
+ "label": "SSL.Error",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_con_latency_ms",
+ "label": "SSL.Connection Latency(ms)",
+ "type": "int"
+ },
+ {
+ "name": "ssl_ja3_fingerprint",
+ "label": "SSL.JA3",
+ "doc": {
+ "visibility": "hidden"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_ja3_hash",
+ "label": "SSL.JA3 hash",
+ "type": "string"
+ },
+ {
+ "name": "ssl_cert_issuer",
+ "label": "SSL.Issuer",
+ "doc": {
+ "constraints": {
+ "type": "items"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssl_cert_subject",
+ "label": "SSL.Subject",
+ "doc": {
+ "constraints": {
+ "type": "items"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "quic_version",
+ "label": "QUIC.Version",
+ "type": "string"
+ },
+ {
+ "name": "quic_sni",
+ "label": "QUIC.SNI",
+ "type": "string"
+ },
+ {
+ "name": "quic_user_agent",
+ "label": "QUIC.User Agent",
+ "type": "string"
+ },
+ {
+ "name": "ftp_account",
+ "label": "FTP.Account",
+ "type": "string"
+ },
+ {
+ "name": "ftp_url",
+ "label": "FTP.URL",
+ "type": "string"
+ },
+ {
+ "name": "ftp_content",
+ "label": "FTP.Content",
+ "type": "string"
+ },
+ {
+ "name": "ftp_link_type",
+ "label": "FTP.Link Type",
+ "type": "string"
+ },
+ {
+ "name": "bgp_type",
+ "label": "BGP.Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "bgp_as_num",
+ "label": "BGP.AS Number",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "bgp_route",
+ "label": "BGP.Route",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "voip_calling_account",
+ "label": "VoIP.Calling Account",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "voip_called_account",
+ "label": "VoIP.Called Account",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "voip_calling_number",
+ "label": "VoIP.Calling Number",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "voip_called_number",
+ "label": "VoIP.Called Number",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "streaming_media_url",
+ "label": "Streaming.Media URL",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "streaming_media_protocol",
+ "label": "Streaming.Media Protocol",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "app_extra_info",
+ "label": "APP.Extra Info",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_call_id",
+ "label": "SIP.Call-ID",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_originator_description",
+ "label": "SIP.Originator",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_responder_description",
+ "label": "SIP.Responder",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_user_agent",
+ "label": "SIP.User-Agent",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_server",
+ "label": "SIP.Server",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_originator_sdp_connect_ip",
+ "label": "SIP.Originator IP",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_originator_sdp_media_port",
+ "label": "SIP.Originator Port",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "sip_originator_sdp_media_type",
+ "label": "SIP.Originator Media Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_originator_sdp_content",
+ "label": "SIP.Originator Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_responder_sdp_connect_ip",
+ "label": "SIP.Responder IP",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_responder_sdp_media_port",
+ "label": "SIP.Responder Port",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "sip_responder_sdp_media_type",
+ "label": "SIP.Responder Media Type",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_responder_sdp_content",
+ "label": "SIP.Responder Content",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "sip_duration_s",
+ "label": "SIP.Duration(s)",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "sip_bye",
+ "label": "SIP.Bye",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "rtp_payload_type_c2s",
+ "label": "RTP.Payload Type(c2s)",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "PCMU"
+ },
+ {
+ "code": "1",
+ "value": "1016"
+ },
+ {
+ "code": "2",
+ "value": "G721"
+ },
+ {
+ "code": "3",
+ "value": "GSM"
+ },
+ {
+ "code": "4",
+ "value": "G723"
+ },
+ {
+ "code": "5",
+ "value": "DVI4_8000"
+ },
+ {
+ "code": "6",
+ "value": "DVI4_16000"
+ },
+ {
+ "code": "7",
+ "value": "LPC"
+ },
+ {
+ "code": "8",
+ "value": "PCMA"
+ },
+ {
+ "code": "9",
+ "value": "G722"
+ },
+ {
+ "code": "10",
+ "value": "L16_STEREO"
+ },
+ {
+ "code": "11",
+ "value": "L16_MONO"
+ },
+ {
+ "code": "12",
+ "value": "QCELP"
+ },
+ {
+ "code": "13",
+ "value": "CN"
+ },
+ {
+ "code": "14",
+ "value": "MPA"
+ },
+ {
+ "code": "15",
+ "value": "G728"
+ },
+ {
+ "code": "16",
+ "value": "DVI4_11025"
+ },
+ {
+ "code": "17",
+ "value": "DVI4_22050"
+ },
+ {
+ "code": "18",
+ "value": "G729"
+ },
+ {
+ "code": "19",
+ "value": "CN_OLD"
+ },
+ {
+ "code": "25",
+ "value": "CELB"
+ },
+ {
+ "code": "26",
+ "value": "JPEG"
+ },
+ {
+ "code": "28",
+ "value": "NV"
+ },
+ {
+ "code": "31",
+ "value": "H261"
+ },
+ {
+ "code": "32",
+ "value": "MPV"
+ },
+ {
+ "code": "33",
+ "value": "MP2T"
+ },
+ {
+ "code": "34",
+ "value": "H263"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "rtp_payload_type_s2c",
+ "label": "RTP.Payload Type(s2c)",
+ "doc": {
+ "visibility": "disabled",
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "PCMU"
+ },
+ {
+ "code": "1",
+ "value": "1016"
+ },
+ {
+ "code": "2",
+ "value": "G721"
+ },
+ {
+ "code": "3",
+ "value": "GSM"
+ },
+ {
+ "code": "4",
+ "value": "G723"
+ },
+ {
+ "code": "5",
+ "value": "DVI4_8000"
+ },
+ {
+ "code": "6",
+ "value": "DVI4_16000"
+ },
+ {
+ "code": "7",
+ "value": "LPC"
+ },
+ {
+ "code": "8",
+ "value": "PCMA"
+ },
+ {
+ "code": "9",
+ "value": "G722"
+ },
+ {
+ "code": "10",
+ "value": "L16_STEREO"
+ },
+ {
+ "code": "11",
+ "value": "L16_MONO"
+ },
+ {
+ "code": "12",
+ "value": "QCELP"
+ },
+ {
+ "code": "13",
+ "value": "CN"
+ },
+ {
+ "code": "14",
+ "value": "MPA"
+ },
+ {
+ "code": "15",
+ "value": "G728"
+ },
+ {
+ "code": "16",
+ "value": "DVI4_11025"
+ },
+ {
+ "code": "17",
+ "value": "DVI4_22050"
+ },
+ {
+ "code": "18",
+ "value": "G729"
+ },
+ {
+ "code": "19",
+ "value": "CN_OLD"
+ },
+ {
+ "code": "25",
+ "value": "CELB"
+ },
+ {
+ "code": "26",
+ "value": "JPEG"
+ },
+ {
+ "code": "28",
+ "value": "NV"
+ },
+ {
+ "code": "31",
+ "value": "H261"
+ },
+ {
+ "code": "32",
+ "value": "MPV"
+ },
+ {
+ "code": "33",
+ "value": "MP2T"
+ },
+ {
+ "code": "34",
+ "value": "H263"
+ }
+ ]
+ },
+ "type": "int"
+ },
+ {
+ "name": "rtp_pcap_path",
+ "label": "RTP.PCAP",
+ "doc": {
+ "visibility": "disabled",
+ "allow_query": "false",
+ "constraints": {
+ "type": "files"
+ }
+ },
+ "type": "string"
+ },
+ {
+ "name": "rtp_originator_dir",
+ "label": "RTP.Direction",
+ "doc": {
+ "constraints": {
+ "operator_functions": "=,!="
+ },
+ "data": [
+ {
+ "code": "0",
+ "value": "unknown"
+ },
+ {
+ "code": "1",
+ "value": "c2s"
+ },
+ {
+ "code": "2",
+ "value": "s2c"
+ }
+ ],
+ "visibility": "disabled"
+ },
+ "type": "int"
+ },
+ {
+ "name": "ssh_version",
+ "label": "SSH.Version",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_auth_success",
+ "label": "SSH.Authentication Result",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_client_version",
+ "label": "SSH.Client Version",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_server_version",
+ "label": "SSH.Server Version",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_cipher_alg",
+ "label": "SSH.Encryption Algorithm",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_mac_alg",
+ "label": "SSH.Signing Algorithm",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_compression_alg",
+ "label": "SSH.Compression Algorithm",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_kex_alg",
+ "label": "SSH. Key Exchange Algorithm",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_host_key_alg",
+ "label": "SSH.Server Host Key Algorithm",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_host_key",
+ "label": "SSH.Server Key Fingerprint",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "ssh_hassh",
+ "label": "SSH.HASSH",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "stratum_cryptocurrency",
+ "label": "Stratum.Cryptocurrency",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "stratum_mining_pools",
+ "label": "Stratum.Mining Pools",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ },
+ {
+ "name": "stratum_mining_program",
+ "label": "Stratum.Mining Program",
+ "doc": {
+ "visibility": "disabled"
+ },
+ "type": "string"
+ }
+ ]
+} \ No newline at end of file
diff --git a/log-stream/start.sh b/log-stream/start.sh
new file mode 100644
index 0000000..a4d0ccf
--- /dev/null
+++ b/log-stream/start.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#启动storm任务脚本
+source /etc/profile
+#任务jar所在目录
+BASE_DIR=`pwd`
+#primordial
+primordial='log-stream-doublewrite-22.04.jar'
+#jar name
+JAR_NAME='log-stream-doublewrite_tmp.jar'
+#并行度
+PARALLELISM=1
+
+yes | cp -r $primordial $JAR_NAME
+
+#cd $BASE_DIR
+jar -xvf $BASE_DIR/$JAR_NAME service_flow_config.properties
+function read_dir(){
+ for file in `ls $1` #注意此处这是两个反引号，表示运行系统命令
+ do
+ if [ -d $1"/"$file ] #注意此处之间一定要加上空格，否则会报错
+ then
+ read_dir $1"/"$file
+ else
+ num=`flink list | grep "$file" | wc -l`
+ if [ $num -eq "0" ];then
+ cat $1$file > $BASE_DIR/service_flow_config.properties
+ jar -uvf $BASE_DIR/$JAR_NAME service_flow_config.properties
+ flink run -d -c com.zdjizhi.topology.LogFlowWriteTopology -p $PARALLELISM $JAR_NAME $file
+ sleep 10
+ fi
+ fi
+done
+}
+if [ $# != 1 ];then
+ echo "usage: ./startall.sh [Configuration path]"
+ exit 1
+fi
+#读取第一个参数 为配置文件目录名称
+read_dir $1
+rm -rf $JAR_NAME
diff --git a/log-stream/stop.sh b/log-stream/stop.sh
new file mode 100644
index 0000000..c3c45dc
--- /dev/null
+++ b/log-stream/stop.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+if [ ! -n "$1" ]
+then
+ echo -e "\033[33mUsage: sh $0 config/\033[0m"
+ exit
+fi
+
+#!/bin/bash
+#flink任务停止脚本
+source /etc/profile
+
+function read_dir(){
+for file in `ls $1` #注意此处这是两个反引号，表示运行系统命令
+do
+ if [ -d "$1$file" ] #注意此处之间一定要加上空格，否则会报错
+ then
+ read_dir $1$file
+ else
+ if [ `flink list | grep "$file" | awk '{print $4}' | wc -l` -ne 0 ]
+ then
+ echo -e "\033[33mstoping job $file\033[0m"
+ flink list | grep "$file" | awk '{print $4}' | xargs flink cancel
+ echo -e "\033[32mstoped job $file\033[0m"
+ fi
+ fi
+done
+}
+#读取第一个参数 为配置文件目录名
+read_dir $1
generated by cgit v1.2.3 (git 2.39.1) at 2025-12-30 11:48:49 +0000